{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gc\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "import warnings\n",
    "from sklearn.preprocessing import scale\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.metrics import roc_auc_score, precision_recall_curve, roc_curve, average_precision_score,accuracy_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from xgboost.sklearn import XGBClassifier\n",
    "import lightgbm as lgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "NF_BoT_IoT = pd.read_csv(\"NF-BoT-IoT.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>IPV4_SRC_ADDR</th>\n",
       "      <th>L4_SRC_PORT</th>\n",
       "      <th>IPV4_DST_ADDR</th>\n",
       "      <th>L4_DST_PORT</th>\n",
       "      <th>PROTOCOL</th>\n",
       "      <th>L7_PROTO</th>\n",
       "      <th>IN_BYTES</th>\n",
       "      <th>OUT_BYTES</th>\n",
       "      <th>IN_PKTS</th>\n",
       "      <th>OUT_PKTS</th>\n",
       "      <th>TCP_FLAGS</th>\n",
       "      <th>FLOW_DURATION_MILLISECONDS</th>\n",
       "      <th>Label</th>\n",
       "      <th>Attack</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>192.168.100.6</td>\n",
       "      <td>52670</td>\n",
       "      <td>192.168.100.1</td>\n",
       "      <td>53</td>\n",
       "      <td>17</td>\n",
       "      <td>5.212</td>\n",
       "      <td>71</td>\n",
       "      <td>126</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4294966</td>\n",
       "      <td>0</td>\n",
       "      <td>Benign</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>192.168.100.6</td>\n",
       "      <td>49160</td>\n",
       "      <td>192.168.100.149</td>\n",
       "      <td>4444</td>\n",
       "      <td>6</td>\n",
       "      <td>0.000</td>\n",
       "      <td>217753000</td>\n",
       "      <td>199100</td>\n",
       "      <td>4521</td>\n",
       "      <td>4049</td>\n",
       "      <td>24</td>\n",
       "      <td>4176249</td>\n",
       "      <td>1</td>\n",
       "      <td>Theft</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>192.168.100.46</td>\n",
       "      <td>3456</td>\n",
       "      <td>192.168.100.5</td>\n",
       "      <td>80</td>\n",
       "      <td>17</td>\n",
       "      <td>0.000</td>\n",
       "      <td>8508021</td>\n",
       "      <td>8918372</td>\n",
       "      <td>9086</td>\n",
       "      <td>9086</td>\n",
       "      <td>0</td>\n",
       "      <td>4175916</td>\n",
       "      <td>0</td>\n",
       "      <td>Benign</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>192.168.100.3</td>\n",
       "      <td>80</td>\n",
       "      <td>192.168.100.55</td>\n",
       "      <td>8080</td>\n",
       "      <td>6</td>\n",
       "      <td>7.000</td>\n",
       "      <td>8442138</td>\n",
       "      <td>9013406</td>\n",
       "      <td>9086</td>\n",
       "      <td>9086</td>\n",
       "      <td>0</td>\n",
       "      <td>4175916</td>\n",
       "      <td>0</td>\n",
       "      <td>Benign</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>192.168.100.46</td>\n",
       "      <td>80</td>\n",
       "      <td>192.168.100.5</td>\n",
       "      <td>80</td>\n",
       "      <td>6</td>\n",
       "      <td>7.000</td>\n",
       "      <td>8374706</td>\n",
       "      <td>0</td>\n",
       "      <td>9086</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4175916</td>\n",
       "      <td>0</td>\n",
       "      <td>Benign</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>600095</th>\n",
       "      <td>192.168.100.46</td>\n",
       "      <td>80</td>\n",
       "      <td>192.168.100.5</td>\n",
       "      <td>80</td>\n",
       "      <td>6</td>\n",
       "      <td>7.000</td>\n",
       "      <td>2330065</td>\n",
       "      <td>0</td>\n",
       "      <td>2523</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4263037</td>\n",
       "      <td>0</td>\n",
       "      <td>Benign</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>600096</th>\n",
       "      <td>192.168.100.5</td>\n",
       "      <td>0</td>\n",
       "      <td>192.168.100.3</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1054423</td>\n",
       "      <td>0</td>\n",
       "      <td>1513</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4263062</td>\n",
       "      <td>0</td>\n",
       "      <td>Benign</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>600097</th>\n",
       "      <td>192.168.100.7</td>\n",
       "      <td>365</td>\n",
       "      <td>192.168.100.3</td>\n",
       "      <td>565</td>\n",
       "      <td>17</td>\n",
       "      <td>0.000</td>\n",
       "      <td>62422</td>\n",
       "      <td>0</td>\n",
       "      <td>1357</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4263062</td>\n",
       "      <td>0</td>\n",
       "      <td>Benign</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>600098</th>\n",
       "      <td>192.168.100.3</td>\n",
       "      <td>50850</td>\n",
       "      <td>13.54.166.67</td>\n",
       "      <td>8883</td>\n",
       "      <td>6</td>\n",
       "      <td>222.178</td>\n",
       "      <td>11300</td>\n",
       "      <td>1664</td>\n",
       "      <td>32</td>\n",
       "      <td>32</td>\n",
       "      <td>24</td>\n",
       "      <td>4264935</td>\n",
       "      <td>0</td>\n",
       "      <td>Benign</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>600099</th>\n",
       "      <td>192.168.100.6</td>\n",
       "      <td>49160</td>\n",
       "      <td>192.168.100.149</td>\n",
       "      <td>4444</td>\n",
       "      <td>6</td>\n",
       "      <td>0.000</td>\n",
       "      <td>40102320</td>\n",
       "      <td>37280</td>\n",
       "      <td>763</td>\n",
       "      <td>590</td>\n",
       "      <td>24</td>\n",
       "      <td>4270068</td>\n",
       "      <td>1</td>\n",
       "      <td>Theft</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>600100 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         IPV4_SRC_ADDR  L4_SRC_PORT    IPV4_DST_ADDR  L4_DST_PORT  PROTOCOL  \\\n",
       "0        192.168.100.6        52670    192.168.100.1           53        17   \n",
       "1        192.168.100.6        49160  192.168.100.149         4444         6   \n",
       "2       192.168.100.46         3456    192.168.100.5           80        17   \n",
       "3        192.168.100.3           80   192.168.100.55         8080         6   \n",
       "4       192.168.100.46           80    192.168.100.5           80         6   \n",
       "...                ...          ...              ...          ...       ...   \n",
       "600095  192.168.100.46           80    192.168.100.5           80         6   \n",
       "600096   192.168.100.5            0    192.168.100.3            0         6   \n",
       "600097   192.168.100.7          365    192.168.100.3          565        17   \n",
       "600098   192.168.100.3        50850     13.54.166.67         8883         6   \n",
       "600099   192.168.100.6        49160  192.168.100.149         4444         6   \n",
       "\n",
       "        L7_PROTO   IN_BYTES  OUT_BYTES  IN_PKTS  OUT_PKTS  TCP_FLAGS  \\\n",
       "0          5.212         71        126        1         1          0   \n",
       "1          0.000  217753000     199100     4521      4049         24   \n",
       "2          0.000    8508021    8918372     9086      9086          0   \n",
       "3          7.000    8442138    9013406     9086      9086          0   \n",
       "4          7.000    8374706          0     9086         0          0   \n",
       "...          ...        ...        ...      ...       ...        ...   \n",
       "600095     7.000    2330065          0     2523         0          0   \n",
       "600096     0.000    1054423          0     1513         0          0   \n",
       "600097     0.000      62422          0     1357         0          0   \n",
       "600098   222.178      11300       1664       32        32         24   \n",
       "600099     0.000   40102320      37280      763       590         24   \n",
       "\n",
       "        FLOW_DURATION_MILLISECONDS  Label  Attack  \n",
       "0                          4294966      0  Benign  \n",
       "1                          4176249      1   Theft  \n",
       "2                          4175916      0  Benign  \n",
       "3                          4175916      0  Benign  \n",
       "4                          4175916      0  Benign  \n",
       "...                            ...    ...     ...  \n",
       "600095                     4263037      0  Benign  \n",
       "600096                     4263062      0  Benign  \n",
       "600097                     4263062      0  Benign  \n",
       "600098                     4264935      0  Benign  \n",
       "600099                     4270068      1   Theft  \n",
       "\n",
       "[600100 rows x 14 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NF_BoT_IoT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "feats = [f for f in NF_BoT_IoT.columns if f not in ['IPV4_SRC_ADDR','IPV4_DST_ADDR','Label','Attack'] ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>L4_SRC_PORT</th>\n",
       "      <th>L4_DST_PORT</th>\n",
       "      <th>PROTOCOL</th>\n",
       "      <th>L7_PROTO</th>\n",
       "      <th>IN_BYTES</th>\n",
       "      <th>OUT_BYTES</th>\n",
       "      <th>IN_PKTS</th>\n",
       "      <th>OUT_PKTS</th>\n",
       "      <th>TCP_FLAGS</th>\n",
       "      <th>FLOW_DURATION_MILLISECONDS</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>600100.000000</td>\n",
       "      <td>600100.000000</td>\n",
       "      <td>600100.000000</td>\n",
       "      <td>600100.000000</td>\n",
       "      <td>6.001000e+05</td>\n",
       "      <td>6.001000e+05</td>\n",
       "      <td>600100.000000</td>\n",
       "      <td>600100.000000</td>\n",
       "      <td>600100.000000</td>\n",
       "      <td>6.001000e+05</td>\n",
       "      <td>600100.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>46528.901980</td>\n",
       "      <td>7949.493863</td>\n",
       "      <td>6.584854</td>\n",
       "      <td>8.956318</td>\n",
       "      <td>9.222368e+03</td>\n",
       "      <td>6.997822e+03</td>\n",
       "      <td>12.433891</td>\n",
       "      <td>5.696411</td>\n",
       "      <td>21.856519</td>\n",
       "      <td>3.468594e+06</td>\n",
       "      <td>0.976906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>12036.388811</td>\n",
       "      <td>14101.151534</td>\n",
       "      <td>2.567061</td>\n",
       "      <td>34.969431</td>\n",
       "      <td>4.734768e+05</td>\n",
       "      <td>8.214194e+05</td>\n",
       "      <td>246.004187</td>\n",
       "      <td>191.421366</td>\n",
       "      <td>8.114914</td>\n",
       "      <td>1.665532e+06</td>\n",
       "      <td>0.150204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.800000e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>39180.000000</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.400000e+01</td>\n",
       "      <td>4.000000e+01</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>4.277976e+06</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>47872.500000</td>\n",
       "      <td>1864.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.400000e+01</td>\n",
       "      <td>4.000000e+01</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>4.294966e+06</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>55307.000000</td>\n",
       "      <td>8009.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>1.120000e+02</td>\n",
       "      <td>4.000000e+01</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>4.294967e+06</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>65535.000000</td>\n",
       "      <td>65535.000000</td>\n",
       "      <td>17.000000</td>\n",
       "      <td>244.000000</td>\n",
       "      <td>2.282235e+08</td>\n",
       "      <td>2.432197e+08</td>\n",
       "      <td>37817.000000</td>\n",
       "      <td>61110.000000</td>\n",
       "      <td>214.000000</td>\n",
       "      <td>4.294967e+06</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         L4_SRC_PORT    L4_DST_PORT       PROTOCOL       L7_PROTO  \\\n",
       "count  600100.000000  600100.000000  600100.000000  600100.000000   \n",
       "mean    46528.901980    7949.493863       6.584854       8.956318   \n",
       "std     12036.388811   14101.151534       2.567061      34.969431   \n",
       "min         0.000000       0.000000       1.000000       0.000000   \n",
       "25%     39180.000000      80.000000       6.000000       0.000000   \n",
       "50%     47872.500000    1864.000000       6.000000       0.000000   \n",
       "75%     55307.000000    8009.000000       6.000000       7.000000   \n",
       "max     65535.000000   65535.000000      17.000000     244.000000   \n",
       "\n",
       "           IN_BYTES     OUT_BYTES        IN_PKTS       OUT_PKTS  \\\n",
       "count  6.001000e+05  6.001000e+05  600100.000000  600100.000000   \n",
       "mean   9.222368e+03  6.997822e+03      12.433891       5.696411   \n",
       "std    4.734768e+05  8.214194e+05     246.004187     191.421366   \n",
       "min    2.800000e+01  0.000000e+00       1.000000       0.000000   \n",
       "25%    4.400000e+01  4.000000e+01       1.000000       1.000000   \n",
       "50%    4.400000e+01  4.000000e+01       1.000000       1.000000   \n",
       "75%    1.120000e+02  4.000000e+01       2.000000       1.000000   \n",
       "max    2.282235e+08  2.432197e+08   37817.000000   61110.000000   \n",
       "\n",
       "           TCP_FLAGS  FLOW_DURATION_MILLISECONDS          Label  \n",
       "count  600100.000000                6.001000e+05  600100.000000  \n",
       "mean       21.856519                3.468594e+06       0.976906  \n",
       "std         8.114914                1.665532e+06       0.150204  \n",
       "min         0.000000                0.000000e+00       0.000000  \n",
       "25%        22.000000                4.277976e+06       1.000000  \n",
       "50%        22.000000                4.294966e+06       1.000000  \n",
       "75%        22.000000                4.294967e+06       1.000000  \n",
       "max       214.000000                4.294967e+06       1.000000  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NF_BoT_IoT.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    586241\n",
       "0     13859\n",
       "Name: Label, dtype: int64"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NF_BoT_IoT['Label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Reconnaissance    470655\n",
       "DDoS               56844\n",
       "DoS                56833\n",
       "Benign             13859\n",
       "Theft               1909\n",
       "Name: Attack, dtype: int64"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NF_BoT_IoT['Attack'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 构建多种模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = LogisticRegression(random_state=2018,tol=1e-6)  # 逻辑回归模型\n",
    "\n",
    "tree = DecisionTreeClassifier(random_state=2018) #决策树模型\n",
    "\n",
    "svm = SVC(probability=True,random_state=2018,tol=1e-6)  # SVM模型\n",
    "\n",
    "forest=RandomForestClassifier(n_estimators=100,random_state=2018) #　随机森林\n",
    "\n",
    "Gbdt=GradientBoostingClassifier(random_state=2018) #CBDT\n",
    "\n",
    "Xgbc=XGBClassifier(random_state=2018)  #Xgbc\n",
    "\n",
    "gbm=lgb.LGBMClassifier(random_state=2018)  #lgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def muti_score(df_X,df_y,model):\n",
    "    warnings.filterwarnings('ignore')\n",
    "    accuracy = cross_val_score(model, df_X, df_y, scoring='accuracy', cv=5)\n",
    "    precision = cross_val_score(model, df_X, df_y, scoring='precision', cv=5)\n",
    "    recall = cross_val_score(model, df_X, df_y, scoring='recall', cv=5)\n",
    "    f1_score = cross_val_score(model, df_X, df_y, scoring='f1', cv=5)\n",
    "    auc = cross_val_score(model, df_X, df_y, scoring='roc_auc', cv=5)\n",
    "    print(\"准确率:\",accuracy.mean())\n",
    "    print(\"精确率:\",precision.mean())\n",
    "    print(\"召回率:\",recall.mean())\n",
    "    print(\"F1_score:\",f1_score.mean())\n",
    "    print(\"AUC:\",auc.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# muti_score(NF_BoT_IoT[feats],NF_BoT_IoT['Label'],lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cross_val_score(gbm, NF_BoT_IoT[feats],NF_BoT_IoT['Label'], scoring='roc_auc', cv=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用lightGBM进行二分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def LGB_model(data_, test_, y_, folds_):\n",
    "    oof_preds = np.zeros(data_.shape[0])\n",
    "    sub_preds = np.zeros(test_.shape[0])\n",
    "    feature_importance_df = pd.DataFrame()\n",
    "    feats = [f for f in NF_BoT_IoT.columns if f not in ['IPV4_SRC_ADDR','IPV4_DST_ADDR','Label','Attack'] ]\n",
    "    for n_fold, (trn_idx, val_idx) in enumerate(folds_.split(data_)):\n",
    "        trn_x, trn_y = data_[feats].iloc[trn_idx], y_.iloc[trn_idx]\n",
    "        val_x, val_y = data_[feats].iloc[val_idx], y_.iloc[val_idx]\n",
    "        \n",
    "        clf=lgb.LGBMClassifier(random_state=2018)  #lgb\n",
    "        \n",
    "        clf.fit(trn_x, trn_y, \n",
    "                eval_set= [(trn_x, trn_y), (val_x, val_y)], \n",
    "                eval_metric='auc', verbose=100, early_stopping_rounds=40  #30\n",
    "               )\n",
    "\n",
    "        oof_preds[val_idx] = clf.predict_proba(val_x, num_iteration=clf.best_iteration_)[:, 1]\n",
    "        sub_preds += clf.predict_proba(test_[feats], num_iteration=clf.best_iteration_)[:, 1] / folds_.n_splits\n",
    "        \n",
    "        fold_importance_df = pd.DataFrame()\n",
    "        fold_importance_df[\"feature\"] = feats\n",
    "        fold_importance_df[\"importance\"] = clf.feature_importances_\n",
    "        fold_importance_df[\"fold\"] = n_fold + 1\n",
    "        feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n",
    "        \n",
    "        print('Fold %2d AUC : %.6f' % (n_fold + 1, roc_auc_score(val_y, oof_preds[val_idx])))\n",
    "        del clf, trn_x, trn_y, val_x, val_y\n",
    "        gc.collect()\n",
    "        \n",
    "    print('Full AUC score %.6f' % roc_auc_score(y_, oof_preds)) \n",
    "\n",
    "    return oof_preds, sub_preds, feature_importance_df\n",
    "    \n",
    "def display_importances(feature_importance_df_):\n",
    "    # Plot feature importances\n",
    "    cols = feature_importance_df_[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(\n",
    "        by=\"importance\", ascending=False)[:50].index\n",
    "    \n",
    "    best_features = feature_importance_df_.loc[feature_importance_df_.feature.isin(cols)]\n",
    "    \n",
    "    plt.figure(figsize=(8,10))\n",
    "    sns.barplot(x=\"importance\", y=\"feature\", \n",
    "                data=best_features.sort_values(by=\"importance\", ascending=False))\n",
    "    plt.title('LightGBM Features (avg over folds)')\n",
    "    plt.tight_layout()\n",
    "    plt.savefig('lgbm_importances.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 40 rounds\n",
      "[100]\ttraining's auc: 0.996064\ttraining's binary_logloss: 0.0208801\tvalid_1's auc: 0.995645\tvalid_1's binary_logloss: 0.0216743\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.996064\ttraining's binary_logloss: 0.0208801\tvalid_1's auc: 0.995645\tvalid_1's binary_logloss: 0.0216743\n",
      "Fold  1 AUC : 0.995645\n",
      "Training until validation scores don't improve for 40 rounds\n",
      "[100]\ttraining's auc: 0.996048\ttraining's binary_logloss: 0.0210335\tvalid_1's auc: 0.9954\tvalid_1's binary_logloss: 0.0216423\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.996048\ttraining's binary_logloss: 0.0210335\tvalid_1's auc: 0.9954\tvalid_1's binary_logloss: 0.0216423\n",
      "Fold  2 AUC : 0.995400\n",
      "Training until validation scores don't improve for 40 rounds\n",
      "[100]\ttraining's auc: 0.996115\ttraining's binary_logloss: 0.020861\tvalid_1's auc: 0.995318\tvalid_1's binary_logloss: 0.0219843\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.996115\ttraining's binary_logloss: 0.020861\tvalid_1's auc: 0.995318\tvalid_1's binary_logloss: 0.0219843\n",
      "Fold  3 AUC : 0.995318\n",
      "Training until validation scores don't improve for 40 rounds\n",
      "[100]\ttraining's auc: 0.996066\ttraining's binary_logloss: 0.0209302\tvalid_1's auc: 0.995325\tvalid_1's binary_logloss: 0.0221295\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.996066\ttraining's binary_logloss: 0.0209302\tvalid_1's auc: 0.995325\tvalid_1's binary_logloss: 0.0221295\n",
      "Fold  4 AUC : 0.995325\n",
      "Training until validation scores don't improve for 40 rounds\n",
      "[100]\ttraining's auc: 0.996081\ttraining's binary_logloss: 0.0208444\tvalid_1's auc: 0.995289\tvalid_1's binary_logloss: 0.0221931\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's auc: 0.996081\ttraining's binary_logloss: 0.0208444\tvalid_1's auc: 0.995289\tvalid_1's binary_logloss: 0.0221931\n",
      "Fold  5 AUC : 0.995289\n",
      "Full AUC score 0.995384\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "\n",
    "data = NF_BoT_IoT.copy()\n",
    "seed = 546789\n",
    "y = data['Label']\n",
    "folds = KFold(n_splits=5, shuffle=True, random_state=seed)\n",
    "X_train1 = data.copy()\n",
    "oof_preds, IntePre, importances = LGB_model(data, X_train1, y, folds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjkAAALICAYAAACdJeHRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeZhlVX23/fsLzUyDAmqBgC1gUBlskEcFHMApmkcFX+epRU0wUdSYgCYBDSZxSIsaUTRiIogjStAg6qMJCBokQgMNDQgKAkKDyCBDMwu/94+zqz0UNZzuqurqXnV/rutcfc7ae6/126f64nxZa52uVBWSJEmtWWumC5AkSZoOhhxJktQkQ44kSWqSIUeSJDXJkCNJkppkyJEkSU0y5EiackmekeTSAc/dJ8k1012TBpfkiUkWzXQdq1KSlya5OsmyJLtNcO6xSf5pnOOVZIcJ+tg1yU9Xtl4NxpAjaaUluTLJc0e2V9VPqmrHKRpj1A+UJK9O8rMkdyT5bff8bUnSd9293YfW7UnOSfKsvusP6D6MPj6i3/279mPHqGefJA90/Q4/vjPJe1zdgt4/AkfMdBGr2BHAQVW1cVWdN92DVdUFwC1JXjzdY81mhhxJa5wkfw18EvgoMAQ8CvhzYG9g3b5TF1bVxsCmwGeBE5Os3Xf8cuBVSeb0tS0AfjFBCdd2H4bDjxn9oBpR/2T72hLYF/j2VPW5OhnnvXoMcNGqrAX4CvDWVTzmrGLIkTTlRs5MJNk9yXndjMo3kxw/cnYmyV93MzLXJXlT13Yg8DrgPcMzJkk2Bf4BeFtVnVBVt1fPeVX1uqq6Z2Q9VfUA8FVgM3qBaNhvgCXAH3fjbQbsBZy0kvf9tCQ/TXJLkvOT7NN37E1Jft69B79K8taufSPg+8BWfTNDW42cwRrlPb0yyXuTXADckWTOBOMf0I17e5IrkrxujNt4HnBuVd3dd+3fJLm8u/biJC/t2tfrxtq579xHJLkrySO71+/pfqbXJvnT8ZZyuvs+KcnNSS5L8md97Xd1P5/hc3dLcmOSdbrXb+7e398l+UGSx/SdW0nenuSXwC9HjLlekmXA2sD5SS7v2p+Q5LTu/i5K8pIx3i+SHNJ3j28ecexPuvfs9iRLkxzcd/g04DlJ1hurb02OIUfStEqyLvAt4Fh6IeNrwEtHnDZEb7bl0cBbgKOSPLyqjqb3f7sL+2ZM9gTWA/5zBWpYm94MzRXA9SMOH9cdA3h11+9DgtIAYzwa+C7wT/Tu82DgP5I8ojvlt8CLgE2ANwGfSLJ7Vd0BvJAHzw5dO+CwrwH+L/AweuFt1PG7IHUk8MKqmksvyC0eo89dgJH7qS4HnkHvZ/QB4MtJtuwC5YldHcNeCZxeVb9N8gLgr4DnAjsAz2J8XwOuAbYCXg58KMlzuvfjTOBlfee+Fjihqu5Lsj/wd8D/BzwC+EnXV7/9gacCT+xvrKp7utk+gCdV1fZdcPoO8EPgkcA7gK8kecgSbHePB9MLh4/r7rXfvwNv7d73nYFT+8ZeCtwHTMnSrh7KkCNpuj0NmAMcWVX3VdWJwFkjzrkP+Ifu+PeAZYz9H/4tgBur6vfDDX2zF3cleWbfuQcnuQW4A/gX4H1Vdf+I/r4F7NPNEC2gF3omslU33vDjlcDrge9V1feq6oGq+i9gEfAnAFX13aq6vJt1Op3eB+gzBhhrPEdW1dVVdddE4wMPADsn2aCqrquqsZZmHgbc3t9QVd+sqmu7fo+nNxvylO7wV3lwyHlt1wa9wHNMVV1UVXfSC0ijSrIN8HTgvVV1d1UtBv4NeMPIcZKEXiAdHuetwIer6ufd34sPAfP7Z3O64zd379VEngZsDHykqu6tqlOBk0fc57Dhe7ywC6yHjzh+H/DEJJtU1e+q6twRx2+n955rGhhyJE23rYCl9eDfBnz1iHNu6g8twJ30PmRGcxOwRfr2VlTVXlX1sO5Y/3/XjujaNwD2AD6a5IX9nXUfet8FDgO2qKozBrina6vqYX2Pb9Db0/GK/vBD70N7S4AkL0zyv91SzC30wscWA4w1nv73cczxuw/fV9Hbt3Rdku8mefwYff4OmNvfkGRBksV9/e7cV/upwAZJntqFivn0giP0fvb9NY78uffbCri5qvoD1lX0ZvcATgD2TLIV8Eyg6M3YDN/7J/vquxlI37UTjT1aLVd3y5yj1fKQc0ec1+9l9H7WVyU5PcmeI47PBW5Zgdq0Agw5kqbbdcCju//7HrbNClxfI16fSW85ab+BO+i5EDiD3vLOSMcBfw18aQXqGulq4Esjws9GVfWRbs/Ff9D7Bs+juuD1PXofxPDQe4Te7NOGfa+HRjlnZHAcdXyAqvpBVT2PXui6BPj8GPdxAfBHwy+64PJ54CBg8672C4dr74LAN+jNcrwWOLkvqFwHbN3X93g/92uBzZL0B6xtgaXdOLfQm/16ZTfO1/qC89X0loT6732Dqur/ivZo7/F4tWyTpP8zcnktI1w34r627T9YVWdX1X70lr2+Te+9Anp7jehtlB/on1vQijPkSJqsdZKs3/cY+e2VM4H7gYPS2xy7H39Y6hjE9cB2wy+6D7sPAJ9J8vIkGydZK8l8YKOxOulmLp7O6N+gOZ3enopPrUBdI30ZeHGSP06ydvde7JNka3ofZOsBNwC/72aTnj/iHjfvlsyGLQb+JMlmSYaAv1zZ8ZM8KslLur0599BbDhy5bDfsv4Ddk6zfvd6IXkC4AXobqOnN5PT7Kr2ZotfxhyUk6H2gv6nbxLsh8P6xiq+qq4GfAh/uat+V3v6sr4wYZwG92ZH+cf4V+NskO3U1bprkFWONNYCf0QuZ70myTnobuF8MfH2Uc78BHJDevy20IfD3wweSrJvkdUk2rar7gNt48Pu+D3DqaJvlNTUMOZIm63vAXX2Pw/sPVtW99DaEvoXetPzr6e1vGPQ/7P9Ob0/DLUm+3fW5kN6G1vfQ29B7PfA54L30PiiHDX8r6w56swDHdOc9SDfTc0pV3TxgTQ/RfUjvR28D7A30ZhcOAdbqZjbeSe8D8Xf0ZiJO6rv2EnobZX/V3edW9GaVzgeu7Go/fmXH7x5/TW+G4mZ6G4DfNkY/19Nbgtqve30x8DF6YfV6ehuTzxhxzXAo2IreN8WG279Pb8Pzj4DLuj5g7J/9a4B5XZ3fAv6+21s07CR6m3uvr6rz+8b5FvDPwNeT3EZvpulBy5Irovs7+5KujxuBzwALup/TyHO/T2+/16n07vHUEae8Abiyq+vP6f39H/Y6egFN0yQPXiaXpOmX5GfAv1bVMTNdix4qyROBLwJPqSn8kEjyBHoBZL0Re7BmnSS7AEdX1cg9OppChhxJ0y69f2n4Unr/Vzz8f6/bVdV1M1qYpl16/6bOd+kte30ReKCq9p/ZqjRbuFwlaVXYkd7Sy630lk1ebsCZNd5Kb/nscnr7Uf5iZsvRbOJMjiRJapIzOZIkqUlT9kvdpDXVFltsUfPmzZvpMiRJK+Gcc865saoeMdoxQ45mvXnz5rFo0aKZLkOStBKSjPxXppdzuUqSJDXJkCNJkppkyJEkSU0y5EiSpCYZciRJUpP8dpVmvZ9fcxNPPuS4mS5DkppwzkcXzHQJyzmTI0mSmmTIkSRJTTLkSJKkJhlyJElSkww5kiSpSYYcSZLUJEOOJElqkiFHkiQ1yZAjSZKaZMiRJElNMuRIkqQmGXIkSVKTDDmSJKlJhhxJktQkQ44kSWqSIUeSJDXJkCNJkppkyJEkSU0y5EiSpCYZciRJUpMMOZIkqUmGHEmS1CRDjiRJapIhp2FJlo1z7OVJKske45yzVpIjk1yYZEmSs5M8tjt2Zdd2QZLTkzym77oF3TUXJbk4ycHjjHFskiuSLE5ybpI9u/YkOSzJL5P8IsmPkuzUd91Dxk/yra6fy5Lc2j1fnGSvFX3vJElrvjkzXYBWvSRzgXcCP5vg1FcBWwG7VtUDSbYG7ug7vm9V3ZjkA8BhwJ8leSHwl8Dzq+raJOsDb5hgnEOq6oQkzwc+B+wKvB3YC3hSVd3ZHTspyU5Vdfdo41fVS7v72wc4uKpeNNAbIkl6iI1++UPWuveOiU8cYcGC/x743KGhIRYuXLjCYwzKkDM7/SOwEBhzhqWzJXBdVT0AUFXXjHHemfRCE8Df0gsY13bX3A18fsC6fgzs0D1/L7BPVd3Z9fPDJD8FXgf8+zjjDyTJgcCBAOvO3XxFLpWkWWGte+9g7XtuW+Hrli5d8WumiyFnlkmyG7BNVZ083jJS5xvA/yR5BnAK8OWqOm+U814AfLt7vjNwzkqW92JgSZJNgI2q6vIRxxcBOz30sgeNP5CqOho4GmCjocfWStQqSU17YN2NVuq6bbeYO/C5Q0NDKzXGoAw5s0iStYBPAAcMcn5VXZNkR+DZ3eOUJK+oqlO6U36U5FHAb+ktV62sjyY5DLgBeMs45wXoDyRTNb4kaYQ7Hvf8lbruuI8umOJKVp4bj2eXufRmWk5LciXwNHr7XMbcfFxV91TV96vqEOBDwP59h/cFHgNcBPxD13YR8OQVrOuQqppfVc+rqgur6jbgjiTbjThvd+DiCcaXJAkw5MwqVXVrVW1RVfOqah7wv8BLqmrRaOcn2T3JVt3ztehtCL5qRJ930dtovCDJZsCHgYVJhrrr1kuyQvtlOh8FjkyyQdfPc4GnA1+dYHxJkgCXq1q3YZL+zcIfr6qPr8D1jwQ+n2S97vVZwKdHnlRV1yX5GvD2qvrHbgnpv5MMLy99YSVq/xTwcHp7dO4HfgPs14WaMcent6lakiRS5Z5LzW4bDT22Hv+GD8x0GZLUhHNW8Z6cJOdU1ajbLlyukiRJTXK5SiTZBfjSiOZ7quqpUzjGUcDeI5o/WVXHTNUYkiT1M+SIqloCzJ/mMd4+nf1LkjSSy1WSJKlJhhxJktQkQ44kSWqSIUeSJDXJkCNJkppkyJEkSU0y5EiSpCYZciRJUpMMOZIkqUmGHEmS1CRDjiRJapIhR5IkNcmQI0mSmmTIkSRJTTLkSJKkJhlyJElSk+bMdAHSTHvC1puz6KMLZroMSdIUcyZHkiQ1yZAjSZKaZMiRJElNMuRIkqQmGXIkSVKTDDmSJKlJhhxJktQkQ44kSWqSIUeSJDXJkCNJkppkyJEkSU0y5EiSpCb5Czo169173UX8+h92mekyJM0S275/yUyXMGs4kyNJkppkyJEkSU0y5EiSpCYZciRJUpMMOZIkqUmGHEmS1CRDjiRJapIhR5IkNcmQI0mSmmTIkSRJTTLkSJKkJhlyJElSkww5kiSpSYYcSZLUJEOOJElqkiFHkiQ1yZAjSZKaZMiRJElNMuRIkqQmGXIkSVKTDDmSJKlJhhxJktQkQ44kSWqSIadhSZaNc+zlSSrJHuOcMy/JXUnOS/LzJGcleWPf8UclOTnJ+UkuTvK9JLskWdw9bk5yRff8vycYY3HXx78mWas7tlOSU5P8Iskvk7wvSbpjByS5obvukiTvTvLHfWMvS3Jp9/y4lX8XJUlrqjkzXYBWvSRzgXcCPxvg9Murarfuuu2AE5OsVVXHAP8A/FdVfbI7vmtVLQHmd6+PBU6uqhMGGGN+kjnAqcD+Sb4PnAT8RVX9MMmGwH8AbwOO6q47vqoOSrI5cCmwW1UNj30acHBVLRrkPZGkqXLEBQ/jxrvHnkOYs2DBCvU3NDTEwoULJ1vWrGTImZ3+EVgIHLwiF1XVr5L8FfAx4BhgS+CHfccvmExRVfX7JD8FdgBeC5xRVT/sjt2Z5CDgNP4QcoavuynJZV09Vw8yVpIDgQMBHr3pOpMpW5Ie5Ma71+L6u8b5eF26dNUVM8sZcmaZJLsB21TVyUlWKOR0zgUe3z0/Cji+Cx//DRxTVddOorYNgecA7weeB5zTf7yqLk+ycZJNRly3LbA+MHDIqqqjgaMBdn30BrWyNUvSSFus/wDw+zGPz9nsMSvU39DQ0CQrmr0MObNIt9flE8ABk+lm+ElV/aBbwnoB8ELgvCQ7V9UNK9jn9kkWAwX8Z1V9P8nzu9ejGW5/VZJ9gR2BP6uqu1dwXEmacgfvesu4x7d9/+mrqBK58Xh2mQvsDJyW5ErgacBJ420+HsVuwM+HX1TVzVX11ap6A3A28MyVqOvyqppfVbtV1eFd20XAg+rqAtWyqrq9azq+qnYCngF8LIn/uyNJWs6QM4tU1a1VtUVVzauqecD/Ai8ZdHNuknnAEcCnutfP7paYhjczbw/8eorK/Qrw9CTP7frfADiS3l6iB6mqM4EvAe+aorElSQ0w5LRtwyTX9D3+aiX62H74K+TAN4BPdd+sAngysCjJBcCZwL9V1dlTUXhV3QXsBxyW5FJgCb2Zok+Pcck/A2/qwpYkSaTKPZea3XZ99AZ18lt3mOkyJM0S275/yUyX0JQk51TVqNsunMmRJElN8ttVIsku9Pa09Lunqp66Jo0hSVI/Q47o/1eK1+QxJEnq53KVJElqkiFHkiQ1yZAjSZKaZMiRJElNMuRIkqQmGXIkSVKTDDmSJKlJhhxJktQkQ44kSWqSIUeSJDXJkCNJkppkyJEkSU0y5EiSpCYZciRJUpMMOZIkqUmGHEmS1KQ5M12ANNPW3XIntn3/opkuQ5I0xZzJkSRJTTLkSJKkJhlyJElSkww5kiSpSYYcSZLUJEOOJElqkiFHkiQ1yZAjSZKaZMiRJElNMuRIkqQmGXIkSVKTDDmSJKlJ/oJOzXqX/PYS9v7U3jNdhsZxxjvOmOkSJK2BnMmRJElNMuRIkqQmGXIkSVKTDDmSJKlJhhxJktQkQ44kSWqSIUeSJDXJkCNJkppkyJEkSU0y5EiSpCYZciRJUpMMOZIkqUmGHEmS1CRDjiRJapIhR5IkNcmQI0mSmmTIkSRJTTLkSJKkJhlyJElSkww5kiSpSYYcSZLUJEOOJElqkiFHkiQ1yZCjFZJkWffnvCSV5B19xz6d5IBxrj02yRVJFie5JMnfJ3lk1zbUd95numOLu8eyJJd2z49Lsk+SW/uOL07y3O7aQ5NclOSCrv2p0/h2SJJWY3NmugCt0X4LvCvJ56rq3gGvOaSqTkiyPnAxcBzwz8ARwOuT7A48HXhyVX0AIMlpwMFVtah7vQ/wk6p6UX/HSfYEXgTsXlX3JNkCWHeyN6lVa50z1iF35kFtC85e8KDXQ0NDLFy4cFWWJWkNZMjRZNwAnAG8Efj8Cl67fvfnHcDRwBuT7At8EDioqu5biXq2BG6sqnsAqurGsU5MciBwIMC6DzcHrU5yZ1jrjgdPMi+9Y+kMVSNpTWbI0WR9BPh+ki8MeP5HkxwG7AAcWVW/BUjyF8CpwElV9eMB+nlGksV9r18G/BB4f5JfAP8NHF9Vp492cVUdTS9csfG2G9eAtWsVqA2LB3jgQW3bPGybB70eGhpCkiZiyNGkVNUVSc4CXjvgJcPLVRsDpyTZq6p+WlWLk1wIfGbAfh6yXAWQ5MnAM4B9geOT/E1VHTtgn1oN3Lf3QyfxjnvHcTNQiaQ1nRuPNRU+BLyXFfj7VFXLgNPo7b8Z9kD3WGlVdX9VnVZVfw8cRG+GR5I0CxlyNGlVdQm9TcQPmVkZS5I5wFOBy6eqjiQ7JnlcX9N84Kqp6l+StGZxuUpT5YPAeQOcN7wnZ13gFODElRxv5J6cfwKuAD6V5GHA74HL6DYXS5JmH0OOVkhVbdz9eSWwc1/7+UwwM1hVB0xwfJ9B2qvqNGDTMbrZa7wxJEmzh8tVkiSpSc7kaMolOQrYe0TzJ6vqmJmoR5I0OxlyNOWq6u0zXYMkSS5XSZKkJhlyJElSkww5kiSpSYYcSZLUJEOOJElqkiFHkiQ1yZAjSZKaZMiRJElNMuRIkqQmGXIkSVKTDDmSJKlJhhxJktQkQ44kSWqSIUeSJDXJkCNJkppkyJEkSU2aM9MFSDPt8Y98PGe844yZLkOSNMWcyZEkSU0y5EiSpCYZciRJUpMMOZIkqUmGHEmS1CRDjiRJapIhR5IkNcmQI0mSmmTIkSRJTTLkSJKkJhlyJElSkww5kiSpSf6CTs16t196Kac/81kzXcZq7Vk/Pn2mS5CkFeZMjiRJapIhR5IkNcmQI0mSmmTIkSRJTTLkSJKkJhlyJElSkww5kiSpSYYcSZLUJEOOJElqkiFHkiQ1yZAjSZKaZMiRJElNMuRIkqQmGXIkSVKTDDmSJKlJhhxJktQkQ44kSWqSIUeSJDXJkCNJkppkyJEkSU0y5EiSpCYZciRJUpMMOZIkqUlzZroArR6SbA0cBTyRXvg9GTgEeC2wR1Ud1HfuacDB3fnrAZsBGwBLu1P2r6orRxnjSuB24H5gbeAw4C7gA8BeVVVJ1gbOAb4JvKK7dBdgSff8C914fwbc0Nf9PsC9wOeBXYEAtwAvqKplK/yGSJLWeIYckSTAicBnq2q/LmgcDXwQuGis66rqqd31BzAiCI1j36q6McmOwA+r6jFJ3gy8Bfg34B3A2VX1wW58kiyrqvl99R4OfKKqjhhxH38LXF9Vu3SvdwTuG+Q90B98ee21uCV5UNu/L1jwkPOGhoZYuHDhqipLklaYIUcAzwburqpjAKrq/iTvBq4A3jdNY24C/K57/m7gf5KcCRwEPGUl+9wSuGr4RVVdOtaJSQ4EDgR41HrrreRwbbol4eYRIYelS0c/WZJWY4YcAexEb4louaq6Lcmvmfq/Iz/qZo62A17ZjXVdkn8BzgTeWVU3D9DPu5O8vnv+u6ral95S1g+TvBw4BfhiVf1ytIur6mh6s1XsOHduTeqOGvOweujbscHWWz+kbWhoaFWUI0krzZAj6O1fGe2DPsDDx7hmZYPB8HLV9sApSU7r9swcBXykqo4dsJ+HLFdV1eIk2wHPB54LnJ1kz6r6+UrWOiu9/v4HHtL2rOOOm4FKJGly/HaVoLfvZo/+hiSbANsA5/HQoLMZcONkBqyqy4Hr6W10pqoeYOWDU3+/y6rqxKp6G/Bl4E8m26ckac1kyBH0lnY2TLIAoNt4/DHgWOBnwN5Jhrpje9D7RtXVkxkwySOBx9K3h2aykuyd5OHd83XpBagp61+StGZxuUp0X91+KfCZJO+jF36/B/xdVd2T5F3A95KsBSwDXtPNvKyMHyW5H1gH+Juqun4l++nfkwOwP7A98Nluz89awHeB/1jJ/iVJa7jUKJsMpdlkx7lz6+jddp/pMlZrz/rx6TNdgiSNKsk5VbXHaMdcrpIkSU1yuUpTLsnP6O3b6feGqloy2vmSJE0HQ46m3PC/hCxJ0kxyuUqSJDXJkCNJkppkyJEkSU0y5EiSpCYZciRJUpMMOZIkqUmGHEmS1CRDjiRJapIhR5IkNcmQI0mSmmTIkSRJTTLkSJKkJhlyJElSkww5kiSpSYYcSZLUJEOOJElq0pyZLkCaaXN33JFn/fj0mS5DkjTFnMmRJElNMuRIkqQmGXIkSVKTDDmSJKlJhhxJktQkQ44kSWqSIUeSJDXJkCNJkppkyJEkSU0y5EiSpCYZciRJUpMMOZIkqUn+gk7Ner+95lY+/dffmeky1igHfezFM12CJE3ImRxJktQkQ44kSWqSIUeSJDXJkCNJkppkyJEkSU0y5EiSpCYZciRJUpMMOZIkqUmGHEmS1CRDjiRJapIhR5IkNcmQI0mSmmTIkSRJTTLkSJKkJhlyJElSkww5kiSpSYYcSZLUJEOOJElqkiFHkiQ1yZAjSZKaZMiRJElNMuRIkqQmGXIkSVKTpi3kJLk/yeK+x7wk+yQ5eZRzd0pyapJfJPllkvflD25M8vDuvC2TVJKn9117Q5LNx6jh8CRLu/F/meTEJE/sO35lki36Xi+vL8kBXd+Lk1yS5N0j+t6tq+WPu9ff6s69LMmtffe9V5LTkuzRnbdpkuOSXN49jkuyaXdsXtfnO/rG+XSSA8Z5n49NcmeSuX1tn+z62aJ7vayv/wvH6OPlI9qWn5tkwyRfSbIkyYVJ/ifJxt2xkT/nv+na10nyke59vzDJWUleONn3oKv1iiTnd39fjkvy6L5z39zVeUE37n5jvXeSpLZN50zOXVU1v+9x5WgnJdkAOAn4SFX9EfAkYC/gbVVVwM+APbvT9wLO6/4kyY7AjVV10zh1fKIb/3HA8cCpSR4x4D0cX1Xzgb2BQ5Ns03fsNcD/dH9SVS/tzv1T4Cd99/3TEX3+O/Crqtq+qrYHrgD+re/4b4F3JVl3wBoBLgP2A0iyFrAvsHQFrp/Iu4Drq2qXqtoZeAtwX3ds5M/5I137PwJbAjt317wYGA5ik30PDqmqJwE70vv78KMk6ybZGjgUeHpV7Qo8DbhgCu5fkrQGmjPTBQCvBc6oqh8CVNWdSQ4CTgOOAs6gF2q+1/35ceBl3bV7ASNDxJiq6vgk/7cb85MrcN1NSS6j96F9dZIALweeB/wkyfpVdfdE/STZAXgy8Kq+5n8ALkuyPXA/cAO9e34j8PkBS/xa1+eXgX2661844LWD2BK4avhFVV063slJNgT+DHhsVd3TXXM98I2pfA+6EPyJJC+ld79XA7cDy7rjy4afa/LOuPxE7rj3NgDOWvDN5e1DQ0MsXLhwpsqSpDFN50zOBn1LGN8a57ydgHP6G6rqcmDjJJvQCzF7dYeeAnwbGJ5R2Yveh+GKOBd4/IpckGRbYH3+MCuwN3BFV+dpwJ8M2NUTgcVVdf9wQ/d8Mb33YdhHgL9OsvaA/f4SeES3rPca4OsDXjeoLwDvTXJmkn9K8ri+Y/0/58VJXgXsAPy6qm4bpa/peA+Gf6bnA9cDVyQ5JsmLx7ogyYFJFiVZtOzOWwcYQnfcext33HMLd9xzC0uXLl3++M1vfjPTpUnSqKZzJueubvlmIgFqjGMFnAXslmQjYJ2qWpbkV92MwF7Ax1awrozof7Qxh70qyb70lkX+rG+2pj9IfB14A3DigGOPNuaD2qvqiiRn0ZtxGtSJwKuBpwJvXYHrJlRVi5NsBzwfeC5wdpI9q+rnjPJzTrLrON1Nx3uQ7pr7k7wA+D/Ac+jN8jy5qg4f5Z6OBo4G2HbocWP9/VOfjdbdZPnzh22x0fLnQ0NDM1GOJE1odViuugh4Zn9D94G6rKpu715fBryZ3hv05koAACAASURBVP+xA/wvvdmTRwLjLp2MYjdgUff8JuDhwI3d6836nkNvT85BSfYEvpvk+/SWUl4GvCTJofQ+YDdPMne43gnudbcka1XVA929rUVvH9LPR5z7IeAE4McD3tfX6b0/X6yqB3oralOnW/o5ETgxyQP03v+RNQ+7DNh2jPdkOt6D3YBTujqHg/FZSf4LOAY4fOI71ET23v7/W/78oI+NOUkmSauN1eEr5F8Bnp7kubB8I/KRQP8i/xnAXwJndq/PpLcZ9n+7D7WBJHkZvdmIr3VNp9GbhaFbFnk98KOR11XVmcCXujGfC5xfVdtU1byqegzwH8D+E41fVZfR2yh7WF/zYcC53bH+cy8BLgZeNMi9VdWv6W26/cwg56+IJHvnD99wW5fektNVY51fVXfS21x85PDm4fS+Gff6qXwP0vNOenuG/l+SrZLs3nfK/PHqlCS1bSZCznOSXDP8oPdBtB9wWJJLgSXA2cCn+645A9iOP4Scc4GtGWzT8bu7vSK/pBdinl1VN3TH/hHYIcn59D54L6O3eXc0/wy8CTgQGLnH6D8YfGnpLcAfpfdV88uBP+raRvNBevc5kKr6XLdPaDw79r//SV7RtX+ur+3MEddsD5yeZAm992kRvXuGh+7JGf521WH0Zr0uTu+r6N/uXsPk34OPdj+zX9Bbmtq3qu4F1gGOSO8r/4vpbW5+1wTvhySpUVmBiRCpSdsOPa7e87qPz3QZaxSXqyStLpKcU1V7jHZsdViukiRJmnKrw8bjSes2AL9iRPM3q+qDM1HPdEhyFL2vrvf7ZFUdMxP1SJK0umsi5HRhpplAM5qqevtM1yBJ0prE5SpJktQkQ44kSWqSIUeSJDXJkCNJkppkyJEkSU0y5EiSpCYZciRJUpMMOZIkqUkThpzuNz2/Psn7u9fbJnnK9JcmSZK08gaZyfkMsCfwmu717cBR01aRJEnSFBjk1zo8tap2T3IeQFX9Lsm601yXJEnSpAwyk3NfkrWBAkjyCOCBaa1KkiRpkgYJOUcC3wIemeSDwP8AH5rWqiRJkiZp3OWqJGsBVwDvAZ4DBNi/qn6+CmqTJElaaeOGnKp6IMnHqmpP4JJVVJMkSdKkDbJc9cMkL0uSaa9GkiRpigzy7aq/AjYCfp/kbnpLVlVVm0xrZZIkSZMwYcipqrmrohBppjxy60056GMvnukyJElTbMKQk+SZo7VX1Y+nvhxJkqSpMchy1SF9z9cHngKcAzx7WiqSJEmaAoMsVz1oHj/JNsDCaatIkiRpCqzMbyG/Bth5qguRJEmaSoPsyfkU3a90oBeK5gPnT2dRkiRJkzXInpxFfc9/D3ytqs6YpnokSZKmxCAh52FV9cn+hiTvGtkmSZK0OhlkT84bR2k7YIrrkCRJmlJjzuQkeQ3wWuCxSU7qOzQXuGm6C5MkSZqM8ZarfgpcB2wBfKyv/XbgguksSpIkabLGDDlVdRVwFbDnqitHkiRpaky4JyfJ05KcnWRZknuT3J/ktlVRnCRJ0soa5NtVnwZeDXwT2ANYAOwwnUVJq9J1V1zOB1//8pkuY7Vx6JdPmOkSJGlKDBJyqKrLkqxdVfcDxyT56TTXJUmSNCmDhJw7k6wLLE6ykN5m5I2mtyxJkqTJGeTfyXlDd95BwB3ANsDLprMoSZKkyRrkt5BflWQDYMuq+sAqqEmSJGnSBvl21YuBxcD/617PH/GPA0qSJK12BlmuOhx4CnALQFUtBuZNX0mSJEmTN0jI+X1V3TrtlUiSJE2hQb5ddWGS1wJrJ3kc8E56v/JBkiRptTXmTE6SL3VPLwd2Au4BvgbcBvzl9JcmSZK08sabyXlykscArwL25cG/pHND4O7pLEySJGkyxgs5/0rvG1XbAYv62gNU1y5JkrRaGnO5qqqOrKonAF+oqu36Ho+tKgOOJElarU347aqq+otVUYgkSdJUGuQr5JIkSWscQ44kSWqSIUeSJDXJkCNJkppkyJEkSU0y5EiSpCYZciRJUpMMOZIkqUmGHEmS1CRDjgBIsnmSxd3jN0mW9r1+T5JLklyY5PwkC7prTktyadd2RpIdx+l/+NzhPl/etS8b55pPdnWsNaL99UkuSHJRN/a/JXlYd+xFSc7r2i9O8tapeYckSWua8X5Bp2aRqroJmA+Q5HBgWVUdkeTPgZcCT6mq25JsCuzfd+nrqmpRkgOBjwIvGWeY11XVonGOL9cFm5cCVwPPBE7r2l8AvBt4YVUtTbI28EbgUUnuAI7uar0myXrAvIHeAElScww5msjfAftW1W0AVXUr8MVRzvsx8JdTOO6+wIXA8cBr6EIOcChwcFUt7eq5H/gCQJLN6P2dvqk7dg9w6RTW1JTzbrqdu+9/4CHtCxYsGPX8oaEhFi5cON1lSdKUMeRoTEnmAnOr6vIBTn8xsGSCc76S5K7u+XO62aOxvAb4GvCfwIeSrFNV9wE7AeeOdkFV3ZzkJOCqJKcAJwNfq6qHfJJ3M08HAmy64QYTlN2mu+9/gLtGCTlLly6dgWokaeoZcjSeADXBOcPB5UrgHROcO9ByVZJ1gT8B3l1Vtyf5GfB84LsjztsF+BIwF/i7qjq+qv60a38ucDDwPOCAkWNU1dH0lrZ49OYPn+gem7T+2qNvydtsaMtR24eGhqazHEmacoYcjanbg3NHku2q6ldjnDbwPpsV8AJgU2BJEoANgTvphZyLgN2BH1XVEmB+kk8Dy6djuvYlSb4EXMEoIUew2+ZzR20/9LjjVnElkjQ9/HaVJvJh4KgkmwAk2aRb6plOrwH+tKrmVdU84LHA85Ns2NVzRJKt+87foKtt4yT79LXPB66a5lolSaspZ3I0kc8CGwNnJ7kPuA/42BT2v2GSa/pefwb4Y2D5V7+r6o4k/wO8uKqOT/II4PvdN6tuobdB+Qf0ltfek+RzwF3AHTiLI0mzVqpm5XYEablHb/7wetsLnzPTZaw2Dv3yCTNdgiQNLMk5VbXHaMdcrpIkSU1yuUpTKsm36O2h6ffeqvrBTNQjSZq9DDmaUlX10pmuQZIkcLlKkiQ1ypAjSZKaZMiRJElNMuRIkqQmGXIkSVKTDDmSJKlJhhxJktQkQ44kSWqSIUeSJDXJkCNJkppkyJEkSU0y5EiSpCYZciRJUpMMOZIkqUmGHEmS1CRDjiRJapIhR5IkNWnOTBcgzbQtH7s9h375hJkuQ5I0xZzJkSRJTTLkSJKkJhlyJElSkww5kiSpSYYcSZLUJEOOJElqkiFHkiQ1yZAjSZKaZMiRJElNMuRIkqQmGXIkSVKTDDmSJKlJ/oJOzXp3X3c7P//gqTNdxsCecOizZ7oESVojOJMjSZKaZMiRJElNMuRIkqQmGXIkSVKTDDmSJKlJhhxJktQkQ44kSWqSIUeSJDXJkCNJkppkyJEkSU0y5EiSpCYZciRJUpMMOZIkqUmGHEmS1CRDjiRJapIhR5IkNcmQI0mSmmTIkSRJTTLkSJKkJhlyJElSkww5kiSpSYYcSZLUJEOOJElqkiFHY0qybJS2TyRZ3D1+keSWca6fl+Su7tyLk/xrkrVGaT8uyTp91z09yVlJLukeB3bth/aNfX/f83d2xw/su+asJE+fjvdFkrRmmDPTBWjNUlXvHn6e5B3AbhNccnlVzU8yBzgV2B84t699beC/gFcCX0kyBHwV2L+qzk2yBfCDJEur6oPAB7uxl1XV/L5aXgS8FXh6Vd2YZHfg20meUlW/mar7nwmfPu+r3HT3rctfr7vg2IecMzQ0xMKFC1dhVZK0+jPkaDJeA/z9ICdW1e+T/BTYgV7IGW6/P8lZwKO7prcDx1bVud3xG5O8Bzgc+O44Q7wXOKSqbuyuOzfJF7v+3jfy5G526ECALTd95CC3MGNuuvtWbrjr5j80LJ25WiRpTWLI0UpJ8hjgsfRmZwY5f0PgOcD7R7SvDzwVeFfXtBPwxRGXL+rax7MTcM4o171xtJOr6mjgaICdH71jTdD3jNp8/U0f9HrdzTZ4yDlDQ0OrqhxJWmMYcrSyXg2cUFX3T3De9kkWAwX8Z1V9P8m8vvbHdf1c0J2f7tyRViaIjNXXGuWg3V77oNdPOPTZM1SJJK1Z3HislfVq4GsDnHd5Vc2vqt2q6vCR7fSWr56W5CVd+0XAHiP6eDJw8QTjXNyd12/3Aa6TJDXKkKMVlmRH4OHAmZPtq6quA/4G+Nuu6SjggCTzu7E2B/4ZmGhX7ULgn7vz6a4/APjMZGuUJK2ZXK7SeDZMck3f649X1cfpbTj+elVN1VLQt4HDkzyjqn6S5PXA55PMpbfk9C9V9Z3xOqiqk5I8GvhpkgJuB17fhShJ0ixkyNGYqmrUmb4Ry07jXX8lsPNE7V1YelLf6x8D/2eCvjcepe2zwGcHqU2S1D6XqyRJUpOcydGkJdkF+NKI5nuq6qkzUY8kSWDI0RSoqiXA/AlPlCRpFXK5SpIkNcmQI0mSmmTIkSRJTTLkSJKkJhlyJElSkww5kiSpSYYcSZLUJEOOJElqkiFHkiQ1yZAjSZKaZMiRJElNMuRIkqQmGXIkSVKTDDmSJKlJhhxJktQkQ44kSWrSnJkuQJpp6285lycc+uyZLkOSNMWcyZEkSU0y5EiSpCYZciRJUpMMOZIkqUmGHEmS1CRDjiRJapIhR5IkNcmQI0mSmmTIkSRJTTLkSJKkJhlyJElSkww5kiSpSf6CTs161157LYcffvgqG29VjiVJs5kzOZIkqUmGHEmS1CRDjiRJapIhR5IkNcmQI0mSmmTIkSRJTTLkSJKkJhlyJElSkww5kiSpSYYcSZLUJEOOJElqkiFHkiQ1yZAjSZKaZMiRJElNMuRIkqQmGXIkSVKTDDmSJKlJhhxJktQkQ44kSWqSIUeSJDXJkCNJkppkyJEkSU0y5EiSpCYZcjRlkizr/pyXpJK8o+/Yp5McMM61xya5IsniJOcm2bOv/eXd882SnJfk+u68Xye5oXu+uBv3zUmWJLkgyYVJ9pvm25YkrabmzHQBatZvgXcl+VxV3TvgNYdU1QlJng98Dth1+ECSTYEfAEdX1We7tgOAParqoO711sChwO5VdWuSjYFHTNkdTcKSJUu45557AFiwYMGDjg0NDbFw4cKZKEuSmmbI0XS5ATgDeCPw+RW89sfADn2vNwa+D3x1OOCM4ZHA7cAygKpaNvx8pCQHAgcCbLrppitY3oq75557uOuuuwBYunTptI8nSTLkaHp9BPh+ki+s4HUvBpb0vf448G9V9YkJrjsfuB64IskpwIlV9Z3RTqyqo4GjAbbaaqtawfpW2Hrrrbf8+WabbfagY0NDQ9M9vCTNSoYcTZuquiLJWcBrB7zko0kOozcL9Ja+9lOB/ZIcUVW/HWe8+5O8APg/wHOATyR5clUdvnJ3MHV22WWX5c8PP/zwmStEkmYRNx5run0IeC+D/V07pKrmV9XzqurCvvavA58Fvpdk7ngdVM9ZVfVh4NXAy1a2cEnSms2Qo2lVVZcAFwMvmmQ//wKcAnwrybqjnZNkqyS79zXNB66azLiSpDWXIUerwgeBrSfbSVW9F7ga+FKS0f7urgMckeSSJIuBVwHvmuy4kqQ1k3tyNGWqauPuzyuBnfvaz2eCQF1VBwzSXlVv6nt5bPcYPnYV8OwVqVmS1C5nciRJUpOcydEqleQoYO8RzZ+sqmNmoh5JUrsMOVqlqurtM12DJGl2cLlKkiQ1yZAjSZKaZMiRJElNMuRIkqQmGXIkSVKTDDmSJKlJhhxJktQkQ44kSWqSIUeSJDXJkCNJkppkyJEkSU0y5EiSpCYZciRJUpMMOZIkqUmGHEmS1CRDjiRJalKqaqZrkGbUHnvsUYsWLZrpMiRJKyHJOVW1x2jHnMmRJElNMuRIkqQmGXIkSVKTDDmSJKlJhhxJktQkQ44kSWqSIUeSJDXJkCNJkppkyJEkSU0y5EiSpCYZciRJUpMMOZIkqUlzZroAaab97nc/5xvffMq09f/KV5w1bX1LksbmTI4kSWqSIUeSJDXJkCNJkppkyJEkSU0y5EiSpCYZciRJUpMMOZIkqUmGHEmS1CRDjiRJapIhR5IkNcmQI0mSmmTIkSRJTTLkSJKkJhlyJElSkww5kiSpSYYcSZLUJEOOJElqkiFHkiQ1yZAjSZKaZMiRJElNMuRIkqQmGXIkSVKTDDl6kCT3J1mc5MIk30yy4Sjt30nysL5rdkpyapJfJPllkvel503dNYuT3JtkSff8I911+ye5IMkl3bH9R9RycHfswiTnJ1nQta+b5F+SXN6N959Jtu67btmqebckSaszQ45Guquq5lfVzsC9wJ+P0n4z8HaAJBsAJwEfqao/Ap4E7AW8raqO6a6ZD1wL7Nu9/pskTwKOAParqscDLwGOSLJr1++fA88DntKN+UwgXS0fAuYCf1RVjwO+DZyYZPi4JEnMmekCtFr7CbDrKO1n9rW/Fjijqn4IUFV3JjkIOA04apy+DwY+VFVXdNddkeTDwCHAG4C/oxeKbuuO3wp8sZtZehPw2Kq6vzt2TJI3A88GTpnE/U6Z7568Abff3vt/iJO/s2B5+9DQEAsXLpypsiRpVjHkaFRJ5gAvBP7fiPa1gecA/9417QSc039OVV2eZOMkmwyHlFHsRG8mp98i4O1J5gJzq+ryUa7bAfj1KP0u6vocKOQkORA4EGCLLdYd5JIVcvvta3Hrrb2Qc+utS6e8f0nSxAw5GmmDJIu75z/hD2FmuH0evVDzX117gBqjr7Hax7puuG28Psc6Nt41Dy2s6mjgaIDtt99o4OsGNXfuA8ufb7zxNsufDw0NTfVQkqQxGHI00l3dHppR25NsCpxMb0/OkcBF9PbLLJdkO2BZVd0+zjgXAXsAF/S17Q5cXFW3JbkjyXZV9asR110GPCbJ3BH97w58Z5AbXBX+74vuWv78la84bgYrkaTZy43HWiHd3ph3AgcnWQf4CvD0JM+F5RuRjwQm2nhyBPC3SeZ1182jtw/nY93xDwNHJdmkO75JkgOr6g7gi8DHu6Uzum9dbQicOjV3KUlqgSFHK6yqzgPOB15dVXcB+wGHJbkUWAKcDXx6gj4WA+8FvpPkEnqzMO/p2gE+C/wIODvJhcDpwJ3dsb8F7gZ+keSXwCuAl1bV8LLThkmu6Xv81dTcuSRpTZI/fC5Is9P2229UH/7ITtPW/ytfcda09S1Js12Sc6pqj9GOOZMjSZKaZMiRJElNMuRIkqQmGXIkSVKTDDmSJKlJhhxJktQkQ44kSWqSIUeSJDXJkCNJkppkyJEkSU0y5EiSpCYZciRJUpMMOZIkqUmGHEmS1CRDjiRJapIhR5IkNcmQI0mSmmTIkSRJTTLkSJKkJhlyJElSk+bMdAHSTHv4w5/AK19x1kyXIUmaYs7kSJKkJhlyJElSkww5kiSpSYYcSZLUJEOOJElqkiFHkiQ1yZAjSZKaZMiRJElNMuRIkqQmGXIkSVKTDDmSJKlJhhxJktQkf0GnZr2Lf3cbTzrhB9PW//kv/+Np61uSNDZnciRJUpMMOZIkqUmGHEmS1CRDjv7/9u4+2I66vuP4+0OCgQChOBQjBBuk4dEG1KgofQC0Yqe0oTNQpQWkY4epAxqxD6NMnT7M1GmZ1ikCoUVUaGWwmUgHymiJplFG7YTIkyGFDA9RiE15mNQQaQkl+faP3SuH5J57Tbg5J9nzfs3cubu/3T2/3/nOvTef7P7OriRJnWTIkSRJnWTIkSRJnWTIkSRJnWTIkSRJnWTIkSRJnWTIkSRJnWTIkSRJnWTIkSRJnWTIkSRJnWTIkSRJnWTIkSRJnWTIkSRJnWTIkSRJnWTIkSRJnWTIkSRJnWTIkSRJnWTIkSRJnWTIkSRJnWTIkSRJnWTI0aSSzElya5KHkzya5Mokr0pyUZKrt9v360kWJFmZ5L4kjyd5ul2+L8ncPn18L8nqJPcnWZZkdk/7oe3ym5NUkk3ta21Msq5d/lqSfZJ8OskD7WutSnLU7q6PJGnPNH3YA9CeLUmAW4Brq2phkmnAdcBfAGv6HVdVb2uPvwhYUFWX/gTdnV5VzyT5JHA58OGeccwHlgJvq6q72rYbgNuramm7fh5wODC/qrYlmQM8t5Nv+RWZ9S9LmLZ508vaLrztph8vz549myuuuGKQQ5KkkWXI0WTOAJ6vqs8DVNXWJJcB64BP7KY+76Qn4ADHAzcCF4wFnD5eC2yoqm3tWNf32zHJxcDFAPseetgrHvCYaZs3MW3Tf7+s7QfbrUuSBsOQo8mcCNzd21BVzyZ5nN3383MWsLpn/Vbg/Kr65iTHLQG+meQXgOXAF6rq3vF2rKrraM5IMfPoY+qVD7mx9aCDd2h73YEzf7w8e/bsqepKkjQJQ44mE2C8EBDgkD7H7GpoWJFkK/Bd4I972r8G/G6SO6pqa7+Dq2p9kmNpzj6dASxPcm5VLd/F8ey0Z3/tN3do+/o5Zw6qe0lSDyceazJrgAW9DUlmAUcC97Jj0Hk18Mwu9nV6VZ1cVRdW1Q972sfm8yye7AWqaktVfaWq/hD4JHD2Lo5FkrSXM+RoMsuBmUkuBGgnHv8NcAOwEji155NQC4AZwBNTPIZtwHnAsUn+vN9OSd6U5PB2eR9gPvD9KR6LJGkv4eUqTaiqKslvAIuTfIImGH8ZuLyqtiRZBHy5DRU/As4bm/g7xePYkmQh8I0kT1bVNePsdhjwmSQz2vW7gKvH2U+SNAJSNWVzLqW90syjj6l5f3XVbnv9+52TI0m7TZK7q2rBeNu8XCVJkjrJy1UaqCQraebt9LqgqlaPt78kSbvKkKOBGrsTsiRJu5uXqyRJUicZciRJUicZciRJUicZciRJUicZciRJUicZciRJUicZciRJUicZciRJUicZciRJUicZciRJUicZciRJUicZciRJUicZciRJUicZciRJUicZciRJUicZciRJUidNH/YApGE74ZBZfOecM4c9DEnSFPNMjiRJ6iRDjiRJ6iRDjiRJ6iRDjiRJ6iRDjiRJ6iRDjiRJ6qRU1bDHIA1Vks3A2mGPYw91KPDMsAexB7M+/Vmb/qzNxHa2Pj9TVT893gbvkyPB2qpaMOxB7ImSfMfa9Gd9+rM2/VmbiU1lfbxcJUmSOsmQI0mSOsmQI8F1wx7AHszaTMz69Gdt+rM2E5uy+jjxWJIkdZJnciRJUicZciRJUicZcjSykrwnydokjyT52LDHM2hJjkyyIsmDSdYkWdS2vzrJV5M83H4/pOeYj7f1WpvkzOGNfjCSTEtyb5Lb23Vr00ryU0mWJnmo/Rl6u/V5SZLL2t+rB5LcnGS/Ua1Pks8leSrJAz1tO12LJG9Osrrd9ukkmaxvQ45GUpJpwDXArwAnAOclOWG4oxq4F4Hfr6rjgVOAS9oafAxYXlXzgOXtOu229wEnAu8BFrd17LJFwIM969bmJVcC/1pVxwEn0dTJ+gBJjgA+DCyoqjcA02je/6jW5waa99VrV2pxLXAxMK/92v41d2DI0ah6K/BIVT1WVS8AXwQWDnlMA1VVG6rqnnZ5M80/UkfQ1OHGdrcbgbPb5YXAF6tqS1WtAx6hqWMnJZkD/CpwfU+ztQGSzAJ+EfgsQFW9UFU/xPr0mg7sn2Q6MBP4T0a0PlV1J7Bxu+adqkWS1wKzqurfq/nE1D/0HNOXIUej6gjgiZ719W3bSEoyF3gjsBJ4TVVtgCYIAYe1u41azf4W+CNgW0+btWm8Hnga+Hx7Oe/6JAdgfQCoqh8Afw08DmwANlXVMqxPr52txRHt8vbtEzLkaFSNdy13JO+nkORA4EvAR6rq2Yl2HaetkzVLchbwVFXd/ZMeMk5bJ2vTmg68Cbi2qt4IPEd7uaGPkapPO79kIXAUcDhwQJLzJzpknLbO1mcS/WqxSzUy5GhUrQeO7FmfQ3M6eaQk2Zcm4NxUVbe0zU+2p4Zpvz/Vto9SzU4Ffj3J92guZZ6R5AtYmzHrgfVVtbJdX0oTeqxP413Auqp6uqr+D7gFeAfWp9fO1mJ9u7x9+4QMORpVq4B5SY5K8iqaiW63DXlMA9V+MuGzwINV9ameTbcB72+X3w/c2tP+viQzkhxFM/HvrkGNd5Cq6uNVNaeq5tL8bPxbVZ2PtQGgqv4LeCLJsW3TO4H/wPqMeRw4JcnM9vfsnTRz3qzPS3aqFu0lrc1JTmlremHPMX35FHKNpKp6McmlwB00n3z4XFWtGfKwBu1U4AJgdZL72rbLgb8EliT5AM0f63MBqmpNkiU0/5i9CFxSVVsHP+yhsjYv+RBwU/ufhMeA36H5j/PI16eqViZZCtxD837vpXlUwYGMYH2S3AycBhyaZD3wJ+za79IHaT6ptT/wlfZr4r59rIMkSeoiL1dJkqROMuRIkqROMuRIkqROMuRIkqROMuRIkqROMuRI0oAl+faA+5ub5LcG2ae0JzDkSNKAVdU7BtVX+4DIuYAhRyPH++RI0oAl+VFVHZjkNODPgCeBk2lu/78aWERzw7Ozq+rRJDcAzwMnAq8BPlpVtyfZD7gWWEBz47SPVtWKJBfRPEF9P+AAmqdgHw+so3ni8z8D/9huA7i0qr7djudPgWeANwB3A+dXVSV5C3Ble8wWmrv4/g/NTd1OA2YA11TV309xuaRd5h2PJWm4TqIJIBtp7hx8fVW9NckimrsKf6Tdby7wS8DRwIokPwtcAlBVP5fkOGBZkmPa/d8OzK+qjW14+YOqOgsgyUzgl6vq+STzgJtpghI0T6M/kea5QN8CTk1yF/BPwHuralWSWcD/Ah+gecL2W5LMAL6VZFlVrdsNdZJ2miFHkoZrVftcHpI8Cixr21cDp/fst6SqtgEPJ3kMOA74eeAqgKp6KMn3gbGQ89Wq2tinz32Bq5OcDGztOQaa5wStb8dzH0242gRsqKpVbV/PttvfDcxPck577ME0zxoy5GiPYMiRpOHaRg9USgAAARZJREFU0rO8rWd9Gy//G7393IICMsHrPjfBtstoLpGdRDM38/k+49najiHj9E/b/qGqumOCvqShceKxJO0dzk2yT5KjgdcDa4E7gd8GaC9Tva5t395m4KCe9YNpzsxso3lI67RJ+n4IOLydl0OSg9oJzXcAH0yy79gYkhwwwetIA+WZHEnaO6wFvkEz8fj32vk0i4G/S7KaZuLxRVW1JdnhBM93gReT3E/zFOfFwJeSnAusYOKzPlTVC0neC1yVZH+a+TjvAq6nuZx1T5pOnwbOnoo3K00FP10lSXu49tNVt1fV0mGPRdqbeLlKkiR1kmdyJElSJ3kmR5IkdZIhR5IkdZIhR5IkdZIhR5IkdZIhR5IkddL/Ax+QMQFpalHvAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "display_importances(importances) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用LightGBM进行多分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "col = [\"Attack\"]\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "lbl = LabelEncoder().fit(data[col])\n",
    "data[col] = lbl.transform(data[col])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "def LGB_model(data_, test_, y_, folds_):\n",
    "    oof_preds = np.zeros(data_.shape[0])\n",
    "    sub_preds = np.zeros(test_.shape[0])\n",
    "    feature_importance_df = pd.DataFrame()\n",
    "    feats = [f for f in NF_BoT_IoT.columns if f not in ['IPV4_SRC_ADDR','IPV4_DST_ADDR','Label','Attack'] ]\n",
    "    for n_fold, (trn_idx, val_idx) in enumerate(folds_.split(data_)):\n",
    "        trn_x, trn_y = data_[feats].iloc[trn_idx], y_.iloc[trn_idx]\n",
    "        val_x, val_y = data_[feats].iloc[val_idx], y_.iloc[val_idx]\n",
    "        \n",
    "        \n",
    "        params = {  \n",
    "            'boosting_type': 'gbdt',  \n",
    "            'objective': 'multiclass',\n",
    "            'metric':'multi_logloss'\n",
    "        }\n",
    "        \n",
    "        clf=lgb.LGBMClassifier(**params,random_state=2018)  #lgb\n",
    "        \n",
    "        clf.fit(trn_x, trn_y, \n",
    "                eval_set= [(trn_x, trn_y), (val_x, val_y)], \n",
    "               verbose=100, early_stopping_rounds=40  #30\n",
    "               )\n",
    "\n",
    "        oof_preds[val_idx] = clf.predict(val_x, num_iteration=clf.best_iteration_)\n",
    "        sub_preds += clf.predict(test_[feats], num_iteration=clf.best_iteration_) / folds_.n_splits\n",
    "        \n",
    "        fold_importance_df = pd.DataFrame()\n",
    "        fold_importance_df[\"feature\"] = feats\n",
    "        fold_importance_df[\"importance\"] = clf.feature_importances_\n",
    "        fold_importance_df[\"fold\"] = n_fold + 1\n",
    "        feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n",
    "        \n",
    "        print('Fold %2d accuracy : %.6f' % (n_fold + 1, accuracy_score(val_y, oof_preds[val_idx])))\n",
    "        del clf, trn_x, trn_y, val_x, val_y\n",
    "        gc.collect()\n",
    "        \n",
    "    print('Full AUC accuracy %.6f' % accuracy_score(y_, oof_preds)) \n",
    "\n",
    "    return oof_preds, sub_preds, feature_importance_df\n",
    "    \n",
    "def display_importances(feature_importance_df_):\n",
    "    # Plot feature importances\n",
    "    cols = feature_importance_df_[[\"feature\", \"importance\"]].groupby(\"feature\").mean().sort_values(\n",
    "        by=\"importance\", ascending=False)[:50].index\n",
    "    \n",
    "    best_features = feature_importance_df_.loc[feature_importance_df_.feature.isin(cols)]\n",
    "    \n",
    "    plt.figure(figsize=(8,10))\n",
    "    sns.barplot(x=\"importance\", y=\"feature\", \n",
    "                data=best_features.sort_values(by=\"importance\", ascending=False))\n",
    "    plt.title('LightGBM Features (avg over folds)')\n",
    "    plt.tight_layout()\n",
    "    plt.savefig('lgbm_importances.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 40 rounds\n",
      "Early stopping, best iteration is:\n",
      "[40]\ttraining's multi_logloss: 0.265889\tvalid_1's multi_logloss: 0.270056\n",
      "Fold  1 accuracy : 0.832453\n",
      "Training until validation scores don't improve for 40 rounds\n",
      "Early stopping, best iteration is:\n",
      "[48]\ttraining's multi_logloss: 0.261132\tvalid_1's multi_logloss: 0.26017\n",
      "Fold  2 accuracy : 0.832178\n",
      "Training until validation scores don't improve for 40 rounds\n",
      "Early stopping, best iteration is:\n",
      "[38]\ttraining's multi_logloss: 0.263818\tvalid_1's multi_logloss: 0.266288\n",
      "Fold  3 accuracy : 0.833211\n",
      "Training until validation scores don't improve for 40 rounds\n",
      "Early stopping, best iteration is:\n",
      "[21]\ttraining's multi_logloss: 0.279798\tvalid_1's multi_logloss: 0.280543\n",
      "Fold  4 accuracy : 0.837969\n",
      "Training until validation scores don't improve for 40 rounds\n",
      "[100]\ttraining's multi_logloss: 0.256667\tvalid_1's multi_logloss: 0.260017\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[100]\ttraining's multi_logloss: 0.256667\tvalid_1's multi_logloss: 0.260017\n",
      "Fold  5 accuracy : 0.823488\n",
      "Full AUC accuracy 0.831860\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "\n",
    "seed = 546789\n",
    "y = data['Attack']\n",
    "folds = KFold(n_splits=5, shuffle=True, random_state=seed)\n",
    "X_train1 = data.copy()\n",
    "oof_preds, IntePre, importances = LGB_model(data, X_train1, y, folds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjgAAALICAYAAABy54rvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeZhkVX3/8fcHhp0BQdQGFUfQoAI6IHFfwC1qVPDnvo1oEkwUNSagJqDBJC4ZQSOKRoyCuKIEDaJGDIgLElmHVRAmgDAQZJFl2IXv74+6rUXTS81099TM6ffrefqZqnPvPedbt+eZ+vQ5p6ZTVUiSJLVkrWEXIEmSNNMMOJIkqTkGHEmS1BwDjiRJao4BR5IkNceAI0mSmmPAkTTjkjw9yYUDnrtrkitmuyYNLsljkpw27DpWpSQvTXJ5kuVJdpri3MOT/PMkxyvJI6bo47FJfr6y9WpqBhxJKy3JpUmeM7a9qn5aVdvN0BjjvpkkeXWSXyS5JclvusdvTZK+6+7s3rBuTnJ6kmf2Xb9n90b0sTH97tG1Hz5BPbsmuafrd/TrO9N8jatbyPsn4MBhF7GKHQjsXVUbV9WZsz1YVZ0N3JDkxbM91lxlwJG0xknyt8AngI8CI8CDgL8Engqs23fq4qraGNgU+AxwdJK1+44vBV6VZF5f2yLgV1OUcGX3Rjj6NdQ3qTH1T7evLYHdgG/PVJ+rk0nu1cOA81ZlLcBXgLes4jHnDAOOpBk3dkYiyc5JzuxmUr6Z5MixszJJ/rabibkqyZu6tr2A1wHvHp0pSbIp8I/AW6vqqKq6uXrOrKrXVdUdY+upqnuArwKb0wtDo/4POAf4k268zYGnAMes5Ot+UpKfJ7khyVlJdu079qYkv+zuwf8meUvXvhHwfWCrvhmhrcbOXI1zTy9N8p4kZwO3JJk3xfh7duPenOSSJK+b4GU8Fzijqm7vu/a9SZZ2156f5KVd+3rdWDv0nfuAJLcleWD3/N3d9/TKJH8+2fJN97qPSXJ9kouT/EVf+23d92f03J2SXJtkne75m7v7+9skP0jysL5zK8nbklwEXDRmzPWSLAfWBs5KsrRrf3SSE7vXd16Sl0xwv0iyb99rfPOYYy/s7tnNSZYl2afv8InAs5OsN1HfWnkGHEmzKsm6wLeAw+kFjK8BLx1z2gi9WZYHA38GHJJks6o6lN5PuYv7ZkqeDKwH/OcK1LA2vZmZS4Crxxw+ojsG8Oqu3/uEpAHGeDDwXeCf6b3OfYD/SPKA7pTfAC8CNgHeBHw8yc5VdQvwAu49K3TlgMO+BvhT4H70gtu443ch6mDgBVU1n16IWzJBnzsCY/dPLQWeTu979AHgy0m27MLk0V0do14J/LiqfpPk+cDfAM8BHgE8k8l9DbgC2Ap4OfChJM/u7sfJwMv6zn0tcFRV3ZVkD+Dvgf8HPAD4addXvz2AJwKP6W+sqju6WT6Ax1XVtl1o+g5wHPBA4O3AV5LcZ9m1e4370AuGj+xea7/PA2/p7vsOwAl9Yy8D7gJmZDlX92bAkTTbngTMAw6uqruq6mjglDHn3AX8Y3f8e8ByJv5Hfwvg2qr63WhD36zFbUme0XfuPkluAG4B/hV4X1XdPaa/bwG7djNDi+gFnqls1Y03+vVK4PXA96rqe1V1T1X9EDgNeCFAVX23qpZ2s00/pvfm+fQBxprMwVV1eVXdNtX4wD3ADkk2qKqrqmqi5Zj7ATf3N1TVN6vqyq7fI+nNgjyhO/xV7h1wXtu1QS/sHFZV51XVrfTC0biSPBR4GvCeqrq9qpYA/w68Yew4SUIvjI6O8xbgw1X1y+7vxYeAhf2zON3x67t7NZUnARsDH6mqO6vqBODYMa9z1OhrPLcLqweMOX4X8Jgkm1TVb6vqjDHHb6Z3zzXDDDiSZttWwLK692/2vXzMOdf1BxbgVnpvMOO5DtgifXspquopVXW/7lj/v2sHdu0bALsAH03ygv7Ouje87wL7A1tU1UkDvKYrq+p+fV/foLeH4xX9wYfeG/aWAElekOR/uuWXG+gFjy0GGGsy/fdxwvG7N95X0dundFWS7yZ51AR9/haY39+QZFGSJX397tBX+wnABkme2AWKhfRCI/S+9/01jv2+99sKuL6q+sPVZfRm9QCOAp6cZCvgGUDRm6kZfe2f6KvveiB910419ni1XN4tbY5Xy33OHXNev5fR+15fluTHSZ485vh84IYVqE0DMuBImm1XAQ/ufuoe9dAVuL7GPD+Z3hLS7gN30HMucBK9JZ2xjgD+FvjSCtQ11uXAl8YEn42q6iPdHov/oPdJnQd1oet79N6E4b6vEXqzThv2PR8Z55yxoXHc8QGq6gdV9Vx6gesC4HMTvI6zgT8afdKFls8BewP372o/d7T2LgR8g97sxmuBY/tCylXAQ/r6nuz7fiWweZL+cLU1sKwb5wZ6s16v7Mb5Wl9ovpzeMlD/a9+gqvo/hj3ePZ6slocm6X+P/H0tY1w15nVt3X+wqk6tqt3pLXV9m969Anp7i+htih/ov1TQijHgSJqudZKs3/c19lMqJwN3A3untxF2d/6wvDGIq4FtRp90b3QfAD6d5OVJNk6yVpKFwEYTddLNWDyN8T8p82N6eyg+uQJ1jfVl4MVJ/iTJ2t292DXJQ+i9ia0HXAP8rptFet6Y13j/bpls1BLghUk2TzIC/PXKjp/kQUle0u3FuYPeEuDYpbpRPwR2TrJ+93wjeuHgGuhtlqY3g9Pvq/RmiF7HH5aNoPdm/qZuw+6GwPsnKr6qLgd+Dny4q/2x9PZjfWXMOIvozYr0j/NvwN8l2b6rcdMkr5horAH8gl7AfHeSddLbrP1i4OvjnPsNYM/0/u+gDYF/GD2QZN0kr0uyaVXdBdzEve/7rsAJ422M1/QZcCRN1/eA2/q+Dug/WFV30tv8+Wf0puJfT28/w6D/qH+e3h6GG5J8u+tzMb3Nq++mt3n3auCzwHvovUmOGv301S30fvo/rDvvXroZnuOr6voBa7qP7g16d3qbXa+hN6uwL7BWN6PxDnpvhr+lNwNxTN+1F9DbFPu/3evcit5s0lnApV3tR67s+N3X39Kbmbie3mbft07Qz9X0lp12756fDxxEL6heTW8T8kljrhkNBFvR+0TYaPv36W1u/hFwcdcHTPy9fw2woKvzW8A/dHuJRh1DbyPv1VV1Vt843wL+Bfh6kpvozTDdaylyRXR/Z1/S9XEt8GlgUfd9Gnvu9+nt7zqB3ms8YcwpbwAu7er6S3p//0e9jl440yzIvZfFJWn2JfkF8G9Vddiwa9F9JXkM8EXgCTWDbxJJHk0vfKw3Zs/VnJNkR+DQqhq7J0czxIAjadal9z8IX0jvp+HRn1q3qaqrhlqYZl16/2fOd+ktdX0RuKeq9hhuVZoLXKKStCpsR2+55UZ6SyUvN9zMGW+ht2S2lN7+k78abjmaK5zBkSRJzXEGR5IkNWfGfkGbtKbaYostasGCBcMuQ5K0Ek4//fRrq+oBY9sNOJrzFixYwGmnnTbsMiRJKyHJ2P89GnCJSpIkNciAI0mSmmPAkSRJzTHgSJKk5hhwJElSc/wUlea8X15xHY/f94hhlyFpEqd/dNGwS9AaxhkcSZLUHAOOJElqjgFHkiQ1x4AjSZKaY8CRJEnNMeBIkqTmGHAkSVJzDDiSJKk5BhxJktQcA44kSWqOAUeSJDXHgCNJkppjwJEkSc0x4EiSpOYYcCRJUnMMOJIkqTkGHEmS1BwDjiRJao4BR5IkNceAI0mSmmPAkSRJzTHgSJKk5hhwJElScww4DUuyfJJjL09SSXaZ5Jy1khyc5Nwk5yQ5NcnDu2OXdm1nJ/lxkof1Xbeou+a8JOcn2WeSMQ5PckmSJUnOSPLkrj1J9k9yUZJfJflRku37rrvP+Em+1fVzcZIbu8dLkjxlRe+dJGnNNm/YBWjVSzIfeAfwiylOfRWwFfDYqronyUOAW/qO71ZV1yb5ALA/8BdJXgD8NfC8qroyyfrAG6YYZ9+qOirJ84DPAo8F3gY8BXhcVd3aHTsmyfZVdft441fVS7vXtyuwT1W9aKAbIs2AjS46jrXuvGXqE7VSFi3672GXoFkwMjLC4sWLZ6VvA87c9E/AYmDCmZXOlsBVVXUPQFVdMcF5J9MLTAB/Ry9cXNldczvwuQHr+gnwiO7xe4Bdq+rWrp/jkvwceB3w+UnGH0iSvYC9ANadf/8VuVQa11p33sLad9w07DKatWyZ91YrxoAzxyTZCXhoVR072dJR5xvAz5I8HTge+HJVnTnOec8Hvt093gE4fSXLezFwTpJNgI2qaumY46cB29/3snuNP5CqOhQ4FGCjkYfXStQq3cs962407BKatvUW84ddgmbByMjIrPVtwJlDkqwFfBzYc5Dzq+qKJNsBz+q+jk/yiqo6vjvlR0keBPyG3hLVyvpokv2Ba4A/m+S8AP1hZKbGl6btlkc+b9glNO2Ijy4adglaw7jJeG6ZT2+G5cQklwJPorevZcKNxlV1R1V9v6r2BT4E7NF3eDfgYcB5wD92becBj1/BuvatqoVV9dyqOreqbgJuSbLNmPN2Bs6fYnxJkgw4c0lV3VhVW1TVgqpaAPwP8JKqOm2885PsnGSr7vFa9Db/Xjamz9vobSpelGRz4MPA4iQj3XXrJVmh/TGdjwIHJ9mg6+c5wNOAr04xviRJLlE1bsMk/RuDP1ZVH1uB6x8IfC7Jet3zU4BPjT2pqq5K8jXgbVX1T92y0X8nGV1S+sJK1P5JYDN6e3LuBv4P2L0LNBOOT28DtSRpjkuV+ys1t2008vB61Bs+MOwyJE3idPfgaAJJTq+q+2y1cIlKkiQ1xyUqkWRH4Etjmu+oqifO4BiHAE8d0/yJqjpspsaQJGmUAUdU1TnAwlke422z2b8kSf1copIkSc0x4EiSpOYYcCRJUnMMOJIkqTkGHEmS1BwDjiRJao4BR5IkNceAI0mSmmPAkSRJzTHgSJKk5hhwJElScww4kiSpOQYcSZLUHAOOJElqjgFHkiQ1x4AjSZKaM2/YBUjD9uiH3J/TPrpo2GVIkmaQMziSJKk5BhxJktQcA44kSWqOAUeSJDXHgCNJkppjwJEkSc0x4EiSpOYYcCRJUnMMOJIkqTkGHEmS1BwDjiRJao4BR5IkNcdftqk5786rzuPX/7jjsMvQGmjr958z7BIkTcAZHEmS1BwDjiRJao4BR5IkNceAI0mSmmPAkSRJzTHgSJKk5hhwJElScww4kiSpOQYcSZLUHAOOJElqjgFHkiQ1x4AjSZKaY8CRJEnNMeBIkqTmGHAkSVJzDDiSJKk5BhxJktQcA44kSWqOAUeSJDXHgCNJkppjwJEkSc0x4EiSpOYYcCRJUnMMOFohSZZ3fy5IUkne3nfsU0n2nOTaw5NckmRJkguS/EOSB3ZtI33nfbo7tqT7Wp7kwu7xEUl2TXJj3/ElSZ7TXbtfkvOSnN21P3EWb4ckaTU1b9gFaI32G+CdST5bVXcOeM2+VXVUkvWB84EjgH8BDgRen2Rn4GnA46vqAwBJTgT2qarTuue7Aj+tqhf1d5zkycCLgJ2r6o4kWwDrTvdFzmUHnn0/rr3dn4MmMm/RomGXoBkwMjLC4sWLh12GZpgBR9NxDXAS8Ebgcyt47frdn7cAhwJvTLIb8EFg76q6ayXq2RK4tqruAKiqayc6MclewF4AD950nZUYam649va1uPo2/5mY0LJlw65A0gT8l0vT9RHg+0m+MOD5H02yP/AI4OCq+g1Akr8CTgCOqaqfDNDP05Ms6Xv+MuA44P1JfgX8N3BkVf14vIur6lB6wYrHPniDGrD2OWeL9e8BfjfsMlZb8zZ/2LBL0AwYGRmZ+iStcQw4mpaquiTJKcBrB7xkdIlqY+D4JE+pqp9X1ZIk5wKfHrCf+yxRASR5PPB0YDfgyCTvrarDB+xTY+zz2BuGXcJqbev3j5ufJa0GXFzXTPgQ8B5W4O9TVS0HTqS332bUPd3XSququ6vqxKr6B2BvejM7kqQ5xoCjaauqC+htGL7PjMpEkswDnggsnak6kmyX5JF9TQuBy2aqf0nSmsMlKs2UDwJnDnDe6B6cdYHjgaNXcryxe3D+GbgE+GSS+9HbOHIx3UZiSdLcYsDRCqmqjbs/LwV26Gs/iylmBKtqzymO7zpIe1WdCGw6QTdPmWwMSdLc4BKVJElqjjM4mnFJDgGeOqb5E1V12DDqkSTNPQYczbiqetuwa5AkzW0uUUmSpOYYcCRJUnMMOJIkqTkGHEmS1BwDjiRJao4BR5IkNceAI0mSmmPAkSRJzTHgSJKk5hhwJElScww4kiSpOQYcSZLUHAOOJElqjgFHkiQ1x4AjSZKaY8CRJEnNmTfsAqRhW3fL7dn6/acNuwxJ0gxyBkeSJDXHgCNJkppjwJEkSc0x4EiSpOYYcCRJUnMMOJIkqTkGHEmS1BwDjiRJao4BR5IkNceAI0mSmmPAkSRJzTHgSJKk5vjLNjXnXfCbC3jqJ5867DLUgJPeftKwS5DUcQZHkiQ1x4AjSZKaY8CRJEnNMeBIkqTmGHAkSVJzDDiSJKk5BhxJktQcA44kSWqOAUeSJDXHgCNJkppjwJEkSc0x4EiSpOYYcCRJUnMMOJIkqTkGHEmS1BwDjiRJao4BR5IkNceAI0mSmmPAkSRJzTHgSJKk5hhwJElScww4kiSpOQYcSZLUnFkLOEnuTrKk72tBkl2THDvOudsnOSHJr5JclOR9+YNrk2zWnbdlkkrytL5rr0ly/wlqOCDJsm78i5IcneQxfccvTbJF3/Pf15dkz67vJUkuSPKuMX3v1NXyJ93zb3XnXpzkxr7X/ZQkJybZpTtv0yRHJFnafR2RZNPu2IKuz7f3jfOpJHtOcp8PT3Jrkvl9bZ/o+tmie768r/9zJ+jj5WPafn9ukg2TfCXJOUnOTfKzJBt3x8Z+n9/bta+T5CPdfT83ySlJXjDde9DVekmSs7q/L0ckeXDfuW/u6jy7G3f3ie6dJKld82ax79uqamF/Q5IFY09KsgFwDPBXVXVckg2B/wDeWlWHJPkF8GTge8BTgDO7P3+WZDvg2qq6bpI6Pl5VB3ZjvQo4IcmOVXXNAK/hyKrauwtQFyY5qqou7469BvhZ9+cPquql3Ri7AvtU1Yv6XmN/n58Hzq2qRd2xDwD/DryiO/4b4J1JPltVdw5QI8DFwO7Al5OsBewGLBvw2kG8E7i6qnbsat4OuKs7dp/vc+efgC2BHarqjiQPAp7ZHZvuPdi3qo5K78b+NfCjJDsADwT2A3auqhu7EPaA6b10rYx1TlqH3JqpT2zMolMXDbuEZoyMjLB48eJhl6E12GwGnEG9Fjipqo4DqKpbk+wNnAgcApxEL9CMBpyPAS/rrn0K8PNBB6qqI5P8aTfmJ1bguuuSXEzvDfvy7o315cBzgZ8mWb+qbp+qnySPAB4PvKqv+R+Bi5NsC9wNXEPvNb8R+NyAJX6t6/PLwK7d9S8Y8NpBbAlcNvqkqi6c7OQupP4F8PCquqO75mrgGzN5D6qqgI8neSm913s5cDOwvDu+fPTxODXuBewFsO5m6072crQScmtY65a5twK+7JaZ/LlC0nTMZsDZIMmS7vElozMc49geOL2/oaqWJtk4ySb0Asz7u0NPAP6B3k/t0As4J61gXWcAj1qRC5JsDawPnN01PZXea1qa5ETghcDRA3T1GGBJVd092lBVd3f3afu+/j8CfD/JFwYs8SJg9/SW8l5DL+jMZMD5AnBct4x1PPDFqrqoO9b/fQb4MPBL4NdVddM4fc3GPRj9nh4LXA1ckuR44Oiq+s54F1TVocChABtvvXENMIZWQG1Y3MM9wy5jlXvo/R467BKaMTIyMuwStIZbpUtUEwgw0RtMAacAOyXZCFinqpYn+d9uJuApwEErWFf/vPl44/a3vSrJbsB2wF/0zdK8Bvh69/jrwBsYLOBM9Frv1V5VlyQ5hd5M06COBl4NPBF4ywpcN6WqWpJkG+B5wHOAU5M8uap+yfhLkY+dpLvZuAfprrk7yfOBPwaeTW925/FVdcAAfWgG3fXUu6Y+qUFHvP2IYZcgqbM6zCGfB+zS39C9mS6vqpur6lZ6e0zeTO8ndYD/oTdr8kBg0uWScexEb4YB4Dpgs75jmwPX9j0/sqq2B54OHJRkJMna9JbI3p/kUuCTwAv6N/lO4jx6Ye339717/Li+mkZ9CHgPg3+Pvk5v38sPq2rGf3SuquVVdXRVvZXeDNELJzn9YmDrCe7JbNyD339Pq+eUqvowvcD3skmvlCQ1aXUIOF8BnpbkOfD7TccHA/27y06ityx1cvf8ZHobX/+n24cxkCQvozcL8bWu6UR6sy90weX1wI/GXldVJwNf6sZ8DnBWVT20qhZU1cPobYreY6rxq+piepuk9+9r3h84ozvWf+4FwPnAixhAVf2a3gbbTw9y/opI8tT84ZNs69JbZrpsovO7UPp54ODu/NFPwL1+Ju9Bet5Bb4/QfyXZKsnOfacsnKxOSVK7hhFwnp3kitEvem9CuwP7J7kQOAc4FfhU3zUnAdvwh4BzBvAQBttg/K7u48sX0Qswz+r7BNU/AY9Icha9N92L6c1OjOdfgDfR25j6rTHH/oPBl5P+DPij9D5OvhT4o65tPB+k9zoHUlWfraqlU5y2Xf/9TzL6yaXP9rWdPOaabYEfJzmH3n06jd5rhm4PTt/XR7r2/eltFj4/vY+bf7t7DtO/Bx/tvme/orcctVv3aat1gAPT+1j/Enobmd85xf2QJDUoKzABIjVp4603rsft+7hhl6EGnPT2Ff3Mg6TpSnJ6Ve0ytn11WKKSJEmaUavD/4MzbUn24w//Sdyob1bVB4dRz2xIcgi9j6f3+0RVHTaMeiRJWp01EXC6INNMmBlPVb1t2DVIkrSmcIlKkiQ1x4AjSZKaY8CRJEnNMeBIkqTmGHAkSVJzDDiSJKk5BhxJktQcA44kSWqOAUeSJDXHgCNJkppjwJEkSc0x4EiSpOYYcCRJUnMMOJIkqTkGHEmS1BwDjiRJas68YRcgDdujHvgoTnr7ScMuQ5I0g5zBkSRJzTHgSJKk5hhwJElScww4kiSpOQYcSZLUHAOOJElqjgFHkiQ1x4AjSZKaY8CRJEnNMeBIkqTmGHAkSVJzDDiSJKk5/rJNzXk3X3ghP37GM4ddhobkmT/58bBLkDQLnMGRJEnNMeBIkqTmGHAkSVJzDDiSJKk5BhxJktQcA44kSWqOAUeSJDXHgCNJkppjwJEkSc0x4EiSpOYYcCRJUnMMOJIkqTkGHEmS1BwDjiRJao4BR5IkNceAI0mSmmPAkSRJzTHgSJKk5hhwJElScww4kiSpOQYcSZLUHAOOJElqjgFHkiQ1x4DTsCTLJzn28iSVZJdJzlmQ5LYkZyb5ZZJTkryx7/iDkhyb5Kwk5yf5XpIdkyzpvq5Pckn3+L+nGGNJ18e/JVmrO7Z9khOS/CrJRUnelyTdsT2TXNNdd0GSdyX5k76xlye5sHt8xMrfRUnSmmjesAvQqpdkPvAO4BcDnL60qnbqrtsGODrJWlV1GPCPwA+r6hPd8cdW1TnAwu754cCxVXXUAGMsTDIPOAHYI8n3gWOAv6qq45JsCPwH8FbgkO66I6tq7yT3By4Edqqq0bFPBPapqtMGuSdaOV9eey1u6GXONdbnFy0adgmrhZGRERYvXjzsMqQZY8CZm/4JWAzssyIXVdX/Jvkb4CDgMGBL4Li+42dPp6iq+l2SnwOPAF4LnFRVx3XHbk2yN3Aifwg4o9ddl+Tirp7LBxkryV7AXgAPWm+96ZQ9p92QcP0aHnBYtmzYFUiaBQacOSbJTsBDq+rYJCsUcDpnAI/qHh8CHNkFj/8GDquqK6dR24bAs4H3A88FTu8/XlVLk2ycZJMx120NrA8MHLCq6lDgUIDt5s+vla15rrtfrfm3boOHPGTYJawWRkZGhl2CNKMMOHNIt7fl48Ce0+lm9EFV/aBbtno+8ALgzCQ7VNU1K9jntkmWAAX8Z1V9P8nzuufjGW1/VZLdgO2Av6iq21dwXE3T6+++Z9glTNszj3CLltQiNxnPLfOBHYATk1wKPAk4ZrKNxuPYCfjl6JOqur6qvlpVbwBOBZ6xEnUtraqFVbVTVR3QtZ0H3KuuLkwtr6qbu6Yjq2p74OnAQUn8EVSSBBhw5pSqurGqtqiqBVW1APgf4CWDbsRNsgA4EPhk9/xZ3bLS6MblbYFfz1C5XwGeluQ5Xf8bAAfT2zt0L1V1MvAl4J0zNLYkaQ1nwGnbhkmu6Pv6m5XoY9vRj4kD3wA+2X2CCuDxwGlJzgZOBv69qk6dicKr6jZgd2D/JBcC59CbIfrUBJf8C/CmLmhJkua4VAObBKXp2G7+/Dp0p52HXYaG5Jk/+fGwS5A0DUlOr6r7bLVwBkeSJDXHT1GJJDvS28PS746qeuKaNIYkSaMMOKL/fx9ek8eQJGmUS1SSJKk5BhxJktQcA44kSWqOAUeSJDXHgCNJkppjwJEkSc0x4EiSpOYYcCRJUnMMOJIkqTkGHEmS1BwDjiRJao4BR5IkNceAI0mSmmPAkSRJzTHgSJKk5hhwJElSc+YNuwBp2OZvtx3P/MmPh12GJGkGOYMjSZKaY8CRJEnNMeBIkqTmGHAkSVJzDDiSJKk5BhxJktQcA44kSWqOAUeSJDXHgCNJkppjwJEkSc0x4EiSpOYYcCRJUnP8ZZua835zxY186m+/M+wyVkt7H/TiYZcgSSvFGRxJktQcA44kSWqOAUeSJDXHgCNJkppjwJEkSc0x4EiSpOYYcCRJUnMMOJIkqTkGHEmS1BwDjiRJao4BR5IkNceAI0mSmmPAkSRJzTHgSJKk5hhwJElScww4kiSpOQYcSZLUHAOOJElqjgFHkiQ1x4AjSZKaY8CRJEnNMeBIkqTmGHAkSVJz5g27AK0ekjwEOAR4DL3geyywL/BaYJeq2rvv3BOBfbrz1wM2BzYAlnWn7FFVl44zxqXAzcDdwNrA/sBtwAeAp1RVJVkbOB34JvCK7tIdgXO6x1/oxvsL4Jq+7ncF7gQ+BzwWCHAD8PyqWr7CN2bDiewAACAASURBVESStEYz4IgkAY4GPlNVu3ch41Dgg8B5E11XVU/srt+TMSFoErtV1bVJtgOOq6qHJXkz8GfAvwNvB06tqg9245NkeVUt7Kv3AODjVXXgmNfxd8DVVbVj93w74K5B7oEkqS0GHAE8C7i9qg4DqKq7k7wLuAR43yyNuQnw2+7xu4CfJTkZ2Bt4wkr2uSVw2eiTqrpwWhWuxk5aejS33HnTrI9zyqJvzvoYY42MjLB48eJVPq6kthhwBLA9vWWh36uqm5L8mpn/O/KjbsZoG+CV3VhXJflX4GTgHVV1/QD9vCvJ67vHv62q3egtXx2X5OXA8cAXq+qi8S5OshewF8Bm8x8wrRc0DLfceRO33HHD7I+zbPbHkKTZYMAR9Par1ATtm01wzXjnD2J0iWpb4PgkJ3Z7ZA4BPlJVhw/Yz32WqKpqSZJtgOcBzwFOTfLkqvrlfYqvOpTeMhxbjzxyZV/L0Gy07iarZJz7bbHRKhmn38jIyCofU1J7DDiC3j6bl/U3JNkEeChwJvCqMedvDlw7nQGrammSq+ltaj6lqu5JMu2g0YWlo4Gjk9wDvBC4T8BZ0z112/+3SsbZ+6AXr5JxJGmm+TFxQW85Z8MkiwC6TcYHAYcDvwCemmSkO7YLvU9OXT6dAZM8EHg4fXtmpivJU5Ns1j1el154mrH+JUlrDmdwRPfx7JcCn07yPnrB93vA31fVHUneCXwvyVrAcuA1VXXPSg73oyR3A+sA762qq1eyn/49OAB7ANsCn+n2+KwFfBf4j5XsX5K0BkvVGrf9QJpRW488st79uo8Nu4zVkktUklZ3SU6vql3GtrtEJUmSmuMSlWZckl/Q26fT7w1Vdc5450uSNNMMOJpxo//DsSRJw+ISlSRJao4BR5IkNceAI0mSmmPAkSRJzTHgSJKk5hhwJElScww4kiSpOQYcSZLUnCkDTnpen+T93fOtkzxh9kuTJElaOYPM4HwaeDLwmu75zcAhs1aRJEnSNA3yqxqeWFU7JzkToKp+m2TdWa5LkiRppQ0yg3NXkrWBAkjyAOCeWa1KkiRpGgYJOAcD3wIemOSDwM+AD81qVZIkSdMw6RJVkrWAS4B3A88GAuxRVb9cBbVJkiStlEkDTlXdk+SgqnoycMEqqkmSJGlaBlmiOi7Jy5Jk1quRJEmaAYN8iupvgI2A3yW5nd4yVVXVJrNamSRJ0kqaMuBU1fxVUYg0LA98yKbsfdCLh12GJGkGTRlwkjxjvPaq+snMlyNJkjR9gyxR7dv3eH3gCcDpwLNmpSJJkqRpGmSJ6l5z90keCiyetYokSZKmaWV+m/gVwA4zXYgkSdJMGWQPzifpfk0DvUC0EDhrNouSJEmajkH24JzW9/h3wNeq6qRZqkeSJGnaBgk496uqT/Q3JHnn2DZJkqTVxSB7cN44TtueM1yHJEnSjJlwBifJa4DXAg9PckzfofnAdbNdmCRJ0sqabInq58BVwBbAQX3tNwNnz2ZRkiRJ0zFhwKmqy4DLgCevunIkSZKmb8o9OEmelOTUJMuT3Jnk7iQ3rYriJEmSVsYgn6L6FPBq4JvALsAi4BGzWZS0Kl11yVI++PqXD7uMe9nvy0cNuwRJWqMNEnCoqouTrF1VdwOHJfn5LNclSZK00gYJOLcmWRdYkmQxvY3HG81uWZIkSStvkP8H5w3deXsDtwAPBV42m0VJkiRNxyC/TfyyJBsAW1bVB1ZBTZIkSdMyyKeoXgwsAf6re75wzH/8J0mStFoZZInqAOAJwA0AVbUEWDB7JUmSJE3PIAHnd1V146xXIkmSNEMG+RTVuUleC6yd5JHAO+j9GgdJkqTV0oQzOEm+1D1cCmwP3AF8DbgJ+OvZL02SJGnlTDaD8/gkDwNeBezGvX/h5obA7bNZmCRJ0sqaLOD8G71PTm0DnNbXHqC6dkmSpNXOhEtUVXVwVT0a+EJVbdP39fCqMtxIkqTV1pSfoqqqv1oVhUiSJM2UQT4mLkmStEYx4EiSpOYYcCRJUnMMOJIkqTkGHEmS1BwDjiRJao4BR5IkNceAI0mSmmPAkSRJzTHgaMYkWd79uSBJJXl737FPJdlzkmsPT3JJkiVJzkjy5L72l3ePN09yZpKru/N+neSa7vGSbtw3JzknydlJzk2y+yy/bEnSasiAo9nyG+CdSdZdgWv2raqFwHuBz/YfSLIp8APg0Kp6UHfe+4Ejq2ph9/x3wH7A06rqscCTgLNn4LVIktYwk/02cWk6rgFOAt4IfG4Fr/0J8Ii+5xsD3we+WlWfmeS6BwI3A8sBqmr56OPZcuZ1N3P73ffMeL+LFi2a8T5X1sjICIsXLx52GZK0Qgw4mk0fAb6f5AsreN2LgXP6nn8M+Peq+vgU150FXA1ckuR44Oiq+s54JybZC9gLYNMNN1jB8v7g9rvv4bZZCDjLli2b8T4laS4x4GjWVNUlSU4BXjvgJR9Nsj+92Z8/62s/Adg9yYFV9ZtJxrs7yfOBPwaeDXw8yeOr6oBxzj0UOBTgwfffrAas7z7WX3t2Vnk3H9lyVvpdGSMjI8MuQZJWmAFHs+1DwFH0lp2msm9VHTVO+9eBnwHfS7JbVd08UQdVVcApwClJfggcBhywwlUPaKf7z5+Vfvc74ohZ6VeS5go3GWtWVdUFwPnAi6bZz78CxwPfmmjjcpKtkuzc17QQuGw640qS1kwGHK0KHwQeMt1Oquo9wOXAl5KM93d3HeDAJBckWQK8CnjndMeVJK150pvRl+auB99/s3rrC5497DLuZb8vj7dSJ0kaK8npVbXL2HZncCRJUnPcZKxVKskhwFPHNH+iqg4bRj2SpDYZcLRKVdXbhl2DJKl9LlFJkqTmGHAkSVJzDDiSJKk5BhxJktQcA44kSWqOAUeSJDXHgCNJkppjwJEkSc0x4EiSpOYYcCRJUnMMOJIkqTkGHEmS1BwDjiRJao4BR5IkNceAI0mSmmPAkSRJzTHgSJKk5swbdgHSsG358G3Z78tHDbsMSdIMcgZHkiQ1x4AjSZKaY8CRJEnNMeBIkqTmGHAkSVJzDDiSJKk5BhxJktQcA44kSWqOAUeSJDXHgCNJkppjwJEkSc0x4EiSpOb4yzY1591+1c388oMnDLuMWffo/Z417BIkaZVxBkeSJDXHgCNJkppjwJEkSc0x4EiSpOYYcCRJUnMMOJIkqTkGHEmS1BwDjiRJao4BR5IkNceAI0mSmmPAkSRJzTHgSJKk5hhwJElScww4kiSpOQYcSZLUHAOOJElqjgFHkiQ1x4AjSZKaY8CRJEnNMeBIkqTmGHAkSVJzDDiSJKk5BhxJktQcA44ASHL/JEu6r/9Lsqzv+buTXJDk3CRnJVnUXXNikgu7tpOSbDdJ/6Pnjvb58q59+STXfKKrY60x7a9PcnaS87qx/z3J/bpjL0pyZtd+fpK3zMwdkiStSeYNuwCtHqrqOmAhQJIDgOVVdWCSvwReCjyhqm5KsimwR9+lr6uq05LsBXwUeMkkw7yuqk4bpJ4u1LwUuBx4BnBi1/584F3AC6pqWZK1gTcCD0pyC3BoV+sVSdYDFgx0A1ZDnzrzq1x3+40z1t+6iw6fsb4mMjIywuLFi2d9HEmaigFHU/l7YLequgmgqm4EvjjOeT8B/noGx90NOBc4EngNXcAB9gP2qaplXT13A18ASLI5vb/T13XH7gAuHK/zLpDtBbDlpg+cwbJnznW338g1t10/cx0um7muJGl1Z8DRhJLMB+ZX1dIBTn8xcM4U53wlyW3d42d3s0YTeQ3wNeA/gQ8lWaeq7gK2B84Y74Kquj7JMcBlSY4HjgW+VlX3jHPuofRme9jhwdvVFHUPxf3X33RG+1t38w1mtL/xjIyMzPoYkjQIA44mE2CqN//R0HIp8PYpzh1oiSrJusALgXdV1c1JfgE8D/jumPN2BL4EzAf+vqqOrKo/79qfA+wDPBfYc6oxV0d77/TaGe3v0fs9a0b7k6TVmQFHE+r23NySZJuq+t8JTht4X80KeD6wKXBOEoANgVvpBZzzgJ2BH1XVOcDCJJ8Cfj890bWfk+RLwCWsoQFHkrTy/BSVpvJh4JAkmwAk2aTbvzKbXgP8eVUtqKoFwMOB5yXZsKvnwCQP6Tt/g662jZPs2te+ELhslmuVJK2GnMHRVD4DbAycmuQu4C7goBnsf8MkV/Q9/zTwJ8DvP95dVbck+Rnw4qo6MskDgO93n6C6gd5m5B/QW1J7d5LPArcBt+DsjSTNSalaLfdXSqvMDg/err751s8Mu4xZ5x4cSS1KcnpV7TK23SUqSZLUHJeoNKOSfIvenpl+76mqHwyjHknS3GTA0YyqqpcOuwZJklyikiRJzTHgSJKk5hhwJElScww4kiSpOQYcSZLUHAOOJElqjgFHkiQ1x4AjSZKaY8CRJEnNMeBIkqTmGHAkSVJzDDiSJKk5BhxJktQcA44kSWqOAUeSJDXHgCNJkpozb9gFSMO2/pbzefR+zxp2GZKkGeQMjiRJao4BR5IkNceAI0mSmmPAkSRJzTHgSJKk5hhwJElScww4kiSpOQYcSZLUHAOOJElqjgFHkiQ1x4AjSZKaY8CRJEnN8Zdtas678sorOeCAA4ZdxioxV16nJDmDI0mSmmPAkSRJzTHgSJKk5hhwJElScww4kiSpOQYcSZLUHAOOJElqjgFHkiQ1x4AjSZKaY8CRJEnNMeBIkqTmGHAkSVJzDDiSJKk5BhxJktQcA44kSWqOAUeSJDXHgCNJkppjwJEkSc0x4EiSpOYYcCRJUnMMOJIkqTkGHEmS1BwDjiRJao4BRxNKsnycto8nWdJ9/SrJDZNcvyDJbd255yf5tyRrjdN+RJJ1+q57WpJTklzQfe3Vte/XN/bdfY/f0R3fq++aU5I8bTbuiyRp9Tdv2AVozVJV7xp9nOTtwE5TXLK0qhYmmQecAOwBnNHXvjbwQ+CVwFeSjABfBfaoqjOSbAH8IMmyqvog8MFu7OVVtbCvlhcBbwGeVlXXJtkZ+HaSJ1TV/83U61/VzjnnHO64444Z62/RokUz1hfAyMgIixcvntE+JWkmGHA0Ha8B/mGQE6vqd0l+DjyCXsAZbb87ySnAg7umtwGHV9UZ3fFrk7wbOAD47iRDvAfYt6qu7a47I8kXu/7eN/bkblZoL4BNN910kJcwFHfccQe33XbbjPW3bNmyGetLklZnBhytlCQPAx5Ob1ZmkPM3BJ4NvH9M+/rAE4F3dk3bA18cc/lpXftktgdOH+e6N453clUdChwKsNVWW9UUfQ/NeuutN6P9bb755jPa38jIyIz2J0kzxYCjlfVq4KiqunuK87ZNsgQo4D+r6vtJFvS1P7Lr5+zu/HTnjrUyIWSivtYYO+6444z2d8ABB8xof5K0unKTsVbWq4GvDXDe0qpaWFU7VdUBY9vpLVk9KclLuvbzgF3G9PF44Pwpxjm/O6/fzgNcJ0lqkAFHKyzJdsBmwMnT7auqrgLeC/xd13QIsGeShd1Y9wf+BZhqJ+ti4F+68+mu3xP49HRrlCSteVyi0mQ2THJF3/OPVdXH6G0u/npVzdTyz7eBA5I8vap+muT1wOeSzKe3zPSvVfWdyTqoqmOSPBj4eZICbgZe3wUoSdIcY8DRhKpq3Bm+MUtNk11/KbDDVO1dUHpc3/OfAH88Rd8bj9P2GeAzg9QmSWqbS1SSJKk5zuBo2pLsCHxpTPMdVfXEYdQjSZIBR9NWVecAC6c8UZKkVcQlKkmS1BwDjiRJao4BR5IkNceAI0mSmmPAkSRJzTHgSJKk5hhwJElScww4kiSpOQYcSZLUHAOOJElqjgFHkiQ1x4AjSZKaY8CRJEnNMeBIkqTmGHAkSVJzDDiSJKk5qaph1yAN1S677FKnnXbasMuQJK2EJKdX1S5j253BkSRJzTHgSJKk5hhwJElScww4kiSpOQYcSZLUHAOOJElqjgFHkiQ1x4AjSZKaY8CRJEnNMeBIkqTmGHAkSVJzDDiSJKk584ZdgDRsv/3tL/nGN5+wSsZ65StOWSXjSNJc5wyOJElqjgFHkiQ1x4AjSZKaY8CRJEnNMeBIkqTmGHAkSVJzDDiSJKk5BhxJktQcA44kSWqOAUeSJDXHgCNJkppjwJEkSc0x4EiSpOYYcCRJUnMMOJIkqTkGHEmS1BwDjiRJao4BR5IkNceAI0mSmmPAkSRJzTHgSJKk5hhwJElScww4upckdydZkuTcJN9MsuE47d9Jcr++a7ZPckKSXyW5KMn70vOm7polSe5Mck73+CPddXskOTvJBd2xPcbUsk937NwkZyVZ1LWvm+RfkyztxvvPJA/pu275qrlbkqTVlQFHY91WVQuragfgTuAvx2m/HngbQJINgGOAj1TVHwGPA54CvLWqDuuuWQhcCezWPX9vkscBBwK7V9WjgJcAByZ5bNfvXwLPBZ7QjfkMIF0tHwLmA39UVY8Evg0cnWT0uCRpjps37AK0Wvsp8Nhx2k/ua38tcFJVHQdQVbcm2Rs4EThkkr73AT5UVZd0112S5MPAvsAbgL+nF4hu6o7fCHyxm1F6E/Dwqrq7O3ZYkjcDzwKOn8brnRHfPXYDbr55/J8djv3OokmvHRkZYfHixbNRliTNKQYcjSvJPOAFwH+NaV8beDbw+a5pe+D0/nOqammSjZNsMhpQxrE9vRmcfqcBb0syH5hfVUvHue4RwK/H6fe0rs+BAk6SvYC9ALbYYt1BLhnYzTevxY03jh9wbrxx2YyOJUkanwFHY22QZEn3+Kf8IciMti+gF2h+2LUHqAn6mqh9outG2ybrc6Jjk11z38KqDgUOBdh2240Gvm4Q8+ffM+GxjTd+6KTXjoyMzGQpkjRnGXA01m3dnplx25NsChxLbw/OwcB59PbH/F6SbYDlVXXzJOOcB+wCnN3XtjNwflXdlOSWJNtU1f+Oue5i4GFJ5o/pf2fgO4O8wNn2py+6bcJjr3zFEauwEkmau9xkrBXS7YV5B7BPknWArwBPS/Ic+P2m44OBqTaSHAj8XZIF3XUL6O27Oag7/mHgkCSbdMc3SbJXVd0CfBH4WLdcRvfpqg2BE2bmVUqS1nQGHK2wqjoTOAt4dVXdBuwO7J/kQuAc4FTgU1P0sQR4D/CdJBfQm315d9cO8BngR8CpSc4Ffgzc2h37O+B24FdJLgJeAby0qkaXmjZMckXf19/MzCuXJK0p8of3BGlu2nbbjerDH9l+lYz1ylecskrGkaS5IsnpVbXL2HZncCRJUnMMOJIkqTkGHEmS1BwDjiRJao4BR5IkNceAI0mSmmPAkSRJzTHgSJKk5hhwJElScww4kiSpOQYcSZLUHAOOJElqjgFHkiQ1x4AjSZKaY8CRJEnNMeBIkqTmGHAkSVJzDDiSJKk5BhxJktQcA44kSWrOvGEXIA3bZps9mle+4pRhlyFJmkHO4EiSpOYYcCRJUnMMOJIkqTkGHEmS1BwDjiRJao4BR5IkNceAI0mSmmPAkSRJzTHgSJKk5hhwJElScww4kiSpOQYcSZLUHH/Zpua88397E4876gcz1t9ZL/+TGetLkrRynMGRJEnNMeBIkqTmGHAkSVJzDDiSJKk5BhxJktQcA44kSWqOAUeSJDXHgCNJkppjwJEkSc0x4EiSpOYYcCRJUnP+f3v3H2tJWd9x/P1hgdWFXQJSXH7ZRQoqtiviVWtprdgqldqACVRpBUnakDZi0cYmVGtCm9RUEpsoiBZtC7amliAGJBrQzSpRG1jQ/QHBLSyLurIBybbLastad7/9Y+amp5f727v37H3O+5VM7pznPHPm+c5s7v3szJwZA44kSWqOAUeSJDXHgCNJkppjwJEkSc0x4EiSpOYYcCRJUnMMOJIkqTkGHEmS1BwDjiRJao4BR5IkNceAI0mSmmPAkSRJzTHgaEZJTkpyW5KHk2xL8pEkhye5LMl1E/p+NclYknuSbEzyvSQ/7Oc3JlkzxToeS7IlyaYkdyVZPdB+bD//iiSVZHf/WbuSbO/nv5LkkCQfTfJA/1kbkpxyoLePJOngc+iwB6CDW5IAtwIfr6rzkywDbgD+GnhwquWq6tX98pcBY1V1xSxWd05VPZXkg8D7gD8ZGMda4Bbg1VV1b992I3BHVd3Sv74YOAFYW1X7k5wE/HiOJU9r1RduZtme3dP2ufT2z8zqs1avXs0111yzEMOSJE1gwNFMXg88U1X/CFBV+5K8B9gOfOAArfNuBsIN8BLgJuCS8XAzheOBnVW1vx/rjqk6JrkcuBzgsGOPm/XAlu3ZzbLd/zFtnx/M8L4k6cAz4GgmLwXuH2yoqqeTfI8D9+/nzcCWgde3AW+vqq/PsNzNwNeT/BqwDvjnqvr2ZB2r6ga6I1GsOPX0mu3A9q08asY+Lzhyxaw+a/Xq1bNdrSRpjgw4mkmAyQJAgKOnWGbWgWGC9Un2AZuBvxho/wrwh0nurKp9Uy1cVTuSvIjuqNPrgXVJLqqqdfMcz7M8/Tu/O2Ofr1547kKtTpI0T15krJk8CIwNNiRZBZwMfJtnh5xjgKfmua5zqurMqrq0qv5zoH38+p3rZ/qAqtpbVV+qqj8DPghcMM+xSJKWMAOOZrIOWJHkUoD+IuMPAzcC9wBnD3zjaQxYDnx/gcewH7gYeFGSv5qqU5KzkpzQzx8CrAW+u8BjkSQtAZ6i0rSqqpK8Bbg+yQfoQvEXgfdV1d4kVwJf7APFj4CLxy/yXeBx7E1yPvC1JE9U1ccm6XYc8Mkky/vX9wLXTdJPktS4VM33cgmpDStOPb1O+9C1C/Z5m7wGR5IWTZL7q2psYrunqCRJUnM8RaVFleQeuut0Bl1SVVsm6y9J0nwYcLSoxu9wLEnSgeQpKkmS1BwDjiRJao4BR5IkNceAI0mSmmPAkSRJzTHgSJKk5hhwJElScww4kiSpOQYcSZLUHAOOJElqjgFHkiQ1x4AjSZKaY8CRJEnNMeBIkqTmGHAkSVJzDDiSJKk5hw57ANKwnXH0Ku678NxhD0OStIA8giNJkppjwJEkSc0x4EiSpOYYcCRJUnMMOJIkqTkGHEmS1JxU1bDHIA1Vkj3A1mGPYwiOBZ4a9iCGZFRrH9W6YXRrH4W6f76qfm5io/fBkWBrVY0NexCLLcl9o1g3jG7to1o3jG7to1o3eIpKkiQ1yIAjSZKaY8CR4IZhD2BIRrVuGN3aR7VuGN3aR7VuLzKWJEnt8QiOJElqjgFHkiQ1x4CjkZXkt5JsTfJIkquGPZ6FluSxJFuSbExyX992TJIvJ3m4/3n0QP8/77fF1iTnDm/kc5fkH5I8meSBgbY515rkFf02eyTJR5NksWuZqylqvzrJD/p9vzHJeQPvNVF7kpOTrE/yUJIHk1zZtze936epu/l9PmdV5eQ0chOwDNgGvBA4HNgEnDHscS1wjY8Bx05ouwa4qp+/CvhQP39Gvw2WA6f022bZsGuYQ62vBc4CHvhZagXuBV4DBPgS8KZh1zbP2q8G3jtJ32ZqB44HzurnVwL/3tfX9H6fpu7m9/lcJ4/gaFS9Cnikqh6tqp8AnwXOH/KYFsP5wE39/E3ABQPtn62qvVW1HXiEbhstCVV1N7BrQvOcak1yPLCqqv6tut/+nx5Y5qA1Re1Taab2qtpZVd/q5/cADwEn0vh+n6buqTRR93wYcDSqTgS+P/B6B9P/kliKCrgryf1JLu/bnl9VO6H7RQkc17e3uD3mWuuJ/fzE9qXqiiSb+1NY46dpmqw9yRrg5cA9jNB+n1A3jNA+nw0DjkbVZOeaW7tnwtlVdRbwJuCdSV47Td9R2B7jpqq1pW3wceBU4ExgJ/Dhvr252pMcCXwOeHdVPT1d10nalmztk9Q9Mvt8tgw4GlU7gJMHXp8EPD6ksRwQVfV4//NJ4PN0p5ye6A9N0/98su/e4vaYa607+vmJ7UtOVT1RVfuqaj/wSf7vdGNTtSc5jO6P/Geq6ta+ufn9Plndo7LP58KAo1G1ATgtySlJDgfeBtw+5DEtmCRHJFk5Pg+8EXiArsZ39N3eAdzWz98OvC3J8iSnAKfRXYC4lM2p1v50xp4kv9x/m+TSgWWWlPE/8L230O17aKj2fpx/DzxUVX878FbT+32qukdhn8/ZsK9ydnIa1gScR/cNhG3A+4c9ngWu7YV035zYBDw4Xh/wPGAd8HD/85iBZd7fb4utLLFvUwD/QndY/n/o/mf6B/OpFRij+8OwDbiO/m7vB/M0Re3/BGwBNtP9gTu+tdqBX6U7pbIZ2NhP57W+36epu/l9PtfJRzVIkqTmeIpKkiQ1x4AjSZKaY8CRJEnNMeBIkqTmGHAkSVJzDDiStMiSfHOR17cmye8t5jqlYTPgSNIiq6pfWax1JTkUWAMYcDRSvA+OJC2yJD+qqiOTvA74S+AJumcI3Up3s7YrgecCF1TVtiQ3As8ALwWeD/xpVd2R5Dl0zyAaA37at69Pchnw28BzgCOAFcBLgO10T9j+PN2N4Y7oh3RFVX2zH8/VwFPALwL3A2+vqkrySuAj/TJ7gd8A/gv4G+B1wHLgY1X1dwu8uaR5OXTYA5CkEfcyuvCxC3gU+FRVvSrJlcC7gHf3/dYAv073QMX1SX4BeCdAVf1SkhfTPT3+9L7/a4C1VbWrDy7vrao3AyRZAbyhqp5Jchrd3ZDH+uVeThekHge+AZyd5F7gX4G3VtWGJKuA/6a7a/LuqnplkuXAN5LcVVXbD8B2kubEgCNJw7WhuucCkWQbcFffvgU4Z6DfzdU9SPHhJI8CL6a7bf+1AFX1nSTfBcYDzperatcU6zwMuC7JmcC+gWWge07Rjn48G+mC1W5gZ1Vt6Nf1dP/+G4G1SS7slz2K7llHBhwNnQFHkoZr78D8/oHX+/n/v6MnXk9QQKb53B9P89576E6LvYzuWsxnphjPvn4MmWT99O3vqqo7p1mXNBReZCxJS8NFSQ5JcSSjkQAAANZJREFUcirdw1S3AncDvw/Qn5p6Qd8+0R5g5cDro+iOyOwHLgGWzbDu7wAn9NfhkGRlf/HyncAfJzlsfAz90+ulofMIjiQtDVuBr9FdZPxH/fUz1wOfSLKF7iLjy6pqb/KsAzubgZ8m2QTcCFwPfC7JRcB6pj/aQ1X9JMlbgWuTPJfu+pvfBD5FdwrrW+lW+kPggoUoVvpZ+S0qSTrI9d+iuqOqbhn2WKSlwlNUkiSpOR7BkSRJzfEIjiRJao4BR5IkNceAI0mSmmPAkSRJzTHgSJKk5vwvbgQPDTUG9V4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "display_importances(importances) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:machine]",
   "language": "python",
   "name": "conda-env-machine-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
